|
| 1 | +/*! |
| 2 | + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 3 | + * SPDX-License-Identifier: Apache-2.0 |
| 4 | + * Zuo |
| 5 | + */ |
| 6 | + |
| 7 | +import assert from 'assert' |
| 8 | +import vscode from 'vscode' |
| 9 | +import { qTestingFramework } from './framework/framework' |
| 10 | +import sinon from 'sinon' |
| 11 | +import { Messenger } from './framework/messenger' |
| 12 | +import { registerAuthHook, using, closeAllEditors } from 'aws-core-vscode/test' |
| 13 | +import { loginToIdC } from './utils/setup' |
| 14 | +import { |
| 15 | + codewhispererDiagnosticSourceLabel, |
| 16 | + invalidFileTypeChatMessage, |
| 17 | + CodeAnalysisScope, |
| 18 | + SecurityScanStep, |
| 19 | +} from 'aws-core-vscode/codewhisperer' |
| 20 | +import path from 'path' |
| 21 | +import { ScanAction, scanProgressMessage } from '../../../src/app/amazonqScan/models/constants' |
| 22 | + |
| 23 | +function getWorkspaceFolder(): string { |
| 24 | + return vscode.workspace.workspaceFolders![0].uri.fsPath |
| 25 | +} |
| 26 | + |
| 27 | +describe('Amazon Q Code Review', function () { |
| 28 | + let framework: qTestingFramework |
| 29 | + let tab: Messenger |
| 30 | + |
| 31 | + function extractAndValidateIssues(reviewString: string): Record<string, number> { |
| 32 | + const issueRegex = /- (\w+): `(\d+) issues?`/g |
| 33 | + const issues: Record<string, number> = { |
| 34 | + Critical: 0, |
| 35 | + High: 0, |
| 36 | + Medium: 0, |
| 37 | + Low: 0, |
| 38 | + Info: 0, |
| 39 | + } |
| 40 | + const foundCategories = new Set<string>() |
| 41 | + |
| 42 | + let match |
| 43 | + while ((match = issueRegex.exec(reviewString)) !== null) { |
| 44 | + const [, severity, count] = match |
| 45 | + if (severity in issues) { |
| 46 | + issues[severity] = parseInt(count, 10) |
| 47 | + foundCategories.add(severity) |
| 48 | + } |
| 49 | + } |
| 50 | + |
| 51 | + const expectedCategories = Object.keys(issues) |
| 52 | + const missingCategories = expectedCategories.filter((category) => !foundCategories.has(category)) |
| 53 | + |
| 54 | + assert.deepStrictEqual( |
| 55 | + missingCategories.length, |
| 56 | + 0, |
| 57 | + `Output chat issue format is not correct or it does not have these categories: ${missingCategories.join(', ')}` |
| 58 | + ) |
| 59 | + return issues |
| 60 | + } |
| 61 | + |
| 62 | + function hasExactlyMatchingSecurityDiagnostic( |
| 63 | + diagnostics: vscode.Diagnostic[], |
| 64 | + code: string, |
| 65 | + message: string, |
| 66 | + startLine: number, |
| 67 | + endLine: number, |
| 68 | + count: number = 1 |
| 69 | + ) { |
| 70 | + const matchingDiagnostics = diagnostics.filter( |
| 71 | + (diagnostic) => |
| 72 | + diagnostic.code === code && |
| 73 | + diagnostic.message === message && |
| 74 | + diagnostic.range.start.line === startLine && |
| 75 | + diagnostic.range.end.line === endLine |
| 76 | + ) |
| 77 | + |
| 78 | + assert.deepEqual(matchingDiagnostics.length, count) |
| 79 | + } |
| 80 | + |
| 81 | + async function waitForChatItems(index: number, waitTimeoutInMs: number = 5000, waitIntervalInMs: number = 1000) { |
| 82 | + await tab.waitForEvent(() => tab.getChatItems().length > index, { |
| 83 | + waitTimeoutInMs: waitTimeoutInMs, |
| 84 | + waitIntervalInMs: waitIntervalInMs, |
| 85 | + }) |
| 86 | + } |
| 87 | + |
| 88 | + async function validateInitialChatMessage() { |
| 89 | + tab.addChatMessage({ command: '/review' }) |
| 90 | + await waitForChatItems(4) |
| 91 | + const fileOrWorkspaceMessage = tab.getChatItems()[4] |
| 92 | + assert.deepStrictEqual(fileOrWorkspaceMessage.type, 'ai-prompt') |
| 93 | + } |
| 94 | + |
| 95 | + async function waitForReviewResults(tab: Messenger): Promise<string> { |
| 96 | + await waitForChatItems(7, 600_000, 10_000) |
| 97 | + const scanResultsMessage = tab.getChatItems()[7] |
| 98 | + assert.deepStrictEqual(scanResultsMessage.type, 'answer') |
| 99 | + |
| 100 | + const scanResultBody = scanResultsMessage.body ?? '' |
| 101 | + assert.notDeepStrictEqual(scanResultBody, '') |
| 102 | + return scanResultBody |
| 103 | + } |
| 104 | + |
| 105 | + before(async function () { |
| 106 | + await using(registerAuthHook('amazonq-test-account'), async () => { |
| 107 | + await loginToIdC() |
| 108 | + }) |
| 109 | + }) |
| 110 | + |
| 111 | + beforeEach(async () => { |
| 112 | + registerAuthHook('amazonq-test-account') |
| 113 | + framework = new qTestingFramework('review', true, []) |
| 114 | + tab = framework.createTab() |
| 115 | + }) |
| 116 | + |
| 117 | + afterEach(async () => { |
| 118 | + await closeAllEditors() |
| 119 | + framework.removeTab(tab.tabID) |
| 120 | + framework.dispose() |
| 121 | + sinon.restore() |
| 122 | + }) |
| 123 | + |
| 124 | + describe('Quick action availability', () => { |
| 125 | + it('Shows /review when code review is enabled', async () => { |
| 126 | + const command = tab.findCommand('/review') |
| 127 | + if (!command.length) { |
| 128 | + assert.fail('Could not find command') |
| 129 | + } |
| 130 | + if (command.length > 1) { |
| 131 | + assert.fail('Found too many commands with the name /review') |
| 132 | + } |
| 133 | + }) |
| 134 | + |
| 135 | + it('Does NOT show /review when code review is NOT enabled', () => { |
| 136 | + framework.dispose() |
| 137 | + framework = new qTestingFramework('review', false, []) |
| 138 | + const tab = framework.createTab() |
| 139 | + const command = tab.findCommand('/review') |
| 140 | + if (command.length > 0) { |
| 141 | + assert.fail('Found command when it should not have been found') |
| 142 | + } |
| 143 | + }) |
| 144 | + }) |
| 145 | + |
| 146 | + describe('/review initial chat output', () => { |
| 147 | + it('Shows appropriate message when /review is entered', async () => { |
| 148 | + tab.addChatMessage({ command: '/review' }) |
| 149 | + |
| 150 | + await waitForChatItems(4) |
| 151 | + const fileOrWorkspaceMessage = tab.getChatItems()[4] |
| 152 | + |
| 153 | + assert.deepStrictEqual(fileOrWorkspaceMessage.type, 'ai-prompt') |
| 154 | + assert.deepStrictEqual( |
| 155 | + fileOrWorkspaceMessage.body, |
| 156 | + 'Would you like to review your active file or the workspace you have open?' |
| 157 | + ) |
| 158 | + }) |
| 159 | + }) |
| 160 | + |
| 161 | + describe('/review entry', () => { |
| 162 | + describe('No file open when review active file', () => { |
| 163 | + it('Shows appropriate message when no file is open', async () => { |
| 164 | + await validateInitialChatMessage() |
| 165 | + |
| 166 | + tab.clickButton(ScanAction.RUN_FILE_SCAN) |
| 167 | + |
| 168 | + await waitForChatItems(5) |
| 169 | + const noFileMessage = tab.getChatItems()[5] |
| 170 | + assert.deepStrictEqual(noFileMessage.type, 'answer') |
| 171 | + assert.deepStrictEqual(noFileMessage.body, invalidFileTypeChatMessage) |
| 172 | + }) |
| 173 | + }) |
| 174 | + |
| 175 | + describe('review insecure file or project', async () => { |
| 176 | + const testFolder = path.join(getWorkspaceFolder(), 'QCAFolder') |
| 177 | + const fileName = 'ProblematicCode.java' |
| 178 | + const filePath = path.join(testFolder, fileName) |
| 179 | + |
| 180 | + beforeEach(async () => { |
| 181 | + await validateInitialChatMessage() |
| 182 | + }) |
| 183 | + |
| 184 | + it('/review file gives correct critical and high security issues', async () => { |
| 185 | + const document = await vscode.workspace.openTextDocument(filePath) |
| 186 | + await vscode.window.showTextDocument(document) |
| 187 | + |
| 188 | + tab.clickButton(ScanAction.RUN_FILE_SCAN) |
| 189 | + |
| 190 | + await waitForChatItems(6) |
| 191 | + const scanningInProgressMessage = tab.getChatItems()[6] |
| 192 | + assert.deepStrictEqual( |
| 193 | + scanningInProgressMessage.body, |
| 194 | + scanProgressMessage(SecurityScanStep.CREATE_SCAN_JOB, CodeAnalysisScope.FILE_ON_DEMAND, fileName) |
| 195 | + ) |
| 196 | + |
| 197 | + const scanResultBody = await waitForReviewResults(tab) |
| 198 | + |
| 199 | + const issues = extractAndValidateIssues(scanResultBody) |
| 200 | + assert.deepStrictEqual( |
| 201 | + issues.Critical >= 1, |
| 202 | + true, |
| 203 | + `critical issue ${issues.Critical} is not larger or equal to 1` |
| 204 | + ) |
| 205 | + |
| 206 | + const uri = vscode.Uri.file(filePath) |
| 207 | + const securityDiagnostics: vscode.Diagnostic[] = vscode.languages |
| 208 | + .getDiagnostics(uri) |
| 209 | + .filter((diagnostic) => diagnostic.source === codewhispererDiagnosticSourceLabel) |
| 210 | + |
| 211 | + // 1 exact critical issue matches |
| 212 | + hasExactlyMatchingSecurityDiagnostic( |
| 213 | + securityDiagnostics, |
| 214 | + 'java-do-not-hardcode-database-password', |
| 215 | + 'CWE-798 - Hardcoded credentials', |
| 216 | + 20, |
| 217 | + 21 |
| 218 | + ) |
| 219 | + }) |
| 220 | + |
| 221 | + it('/review project gives findings', async () => { |
| 222 | + tab.clickButton(ScanAction.RUN_PROJECT_SCAN) |
| 223 | + |
| 224 | + const scanResultBody = await waitForReviewResults(tab) |
| 225 | + extractAndValidateIssues(scanResultBody) |
| 226 | + }) |
| 227 | + }) |
| 228 | + |
| 229 | + describe('/review file and project scans should respect ignored line findings', async () => { |
| 230 | + const testFolder = path.join(getWorkspaceFolder(), 'QCAFolder') |
| 231 | + const fileName = 'ProblematicCode.java' |
| 232 | + const filePath = path.join(testFolder, fileName) |
| 233 | + |
| 234 | + beforeEach(async () => { |
| 235 | + await validateInitialChatMessage() |
| 236 | + |
| 237 | + const document = await vscode.workspace.openTextDocument(filePath) |
| 238 | + await vscode.window.showTextDocument(document) |
| 239 | + |
| 240 | + const editor = vscode.window.activeTextEditor |
| 241 | + |
| 242 | + if (editor) { |
| 243 | + const position = new vscode.Position(20, 0) |
| 244 | + await editor.edit((editBuilder) => { |
| 245 | + editBuilder.insert(position, '// amazonq-ignore-next-line\n') |
| 246 | + }) |
| 247 | + } |
| 248 | + }) |
| 249 | + |
| 250 | + it('/review file respect ignored line findings', async () => { |
| 251 | + tab.clickButton(ScanAction.RUN_FILE_SCAN) |
| 252 | + }) |
| 253 | + |
| 254 | + it('/review project respect ignored line findings', async () => { |
| 255 | + tab.clickButton(ScanAction.RUN_PROJECT_SCAN) |
| 256 | + }) |
| 257 | + |
| 258 | + afterEach(async () => { |
| 259 | + await waitForReviewResults(tab) |
| 260 | + |
| 261 | + const uri = vscode.Uri.file(filePath) |
| 262 | + const securityDiagnostics: vscode.Diagnostic[] = vscode.languages |
| 263 | + .getDiagnostics(uri) |
| 264 | + .filter((diagnostic) => diagnostic.source === codewhispererDiagnosticSourceLabel) |
| 265 | + |
| 266 | + // cannot find this ignored issue |
| 267 | + hasExactlyMatchingSecurityDiagnostic( |
| 268 | + securityDiagnostics, |
| 269 | + 'java-do-not-hardcode-database-password', |
| 270 | + 'CWE-798 - Hardcoded credentials', |
| 271 | + 21, |
| 272 | + 22, |
| 273 | + 0 |
| 274 | + ) |
| 275 | + |
| 276 | + const editor = vscode.window.activeTextEditor |
| 277 | + if (editor) { |
| 278 | + await editor.edit((editBuilder) => { |
| 279 | + const lineRange = editor.document.lineAt(20).rangeIncludingLineBreak |
| 280 | + editBuilder.delete(lineRange) |
| 281 | + }) |
| 282 | + } |
| 283 | + }) |
| 284 | + }) |
| 285 | + }) |
| 286 | +}) |
0 commit comments