fix(amazonq): noisy logs for file scans aws#4852

Problem
File scans logs are too noisy

Solution
Use NullLogger for file scans except for errors

Author: ctlai95

packages/core/src/codewhisperer/commands/startSecurityScan.ts

@@ -18,6 +18,7 @@ import {
     pollScanJobStatus,
     listScanResults,
     throwIfCancelled,
+    getLoggerForScope,
 } from '../service/securityScanHandler'
 import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
 import { AggregatedCodeScanIssue, CodeScansState, codeScanState, CodeScanTelemetryEntry } from '../models/model'
@@ -89,6 +90,7 @@ export async function startSecurityScan(
     context: vscode.ExtensionContext,
     scope: CodeWhispererConstants.CodeAnalysisScope
 ) {
+    const logger = getLoggerForScope(scope)
     /**
      * Step 0: Initial Code Scan telemetry
      */
@@ -113,7 +115,7 @@ export async function startSecurityScan(
         codewhispererCodeScanScope: scope,
     }
     try {
-        getLogger().verbose(`Starting security scan `)
+        logger.verbose(`Starting security scan `)
         /**
          * Step 1: Generate zip
          */
@@ -124,7 +126,7 @@ export async function startSecurityScan(
 
         const contextTruncationStartTime = performance.now()
         codeScanTelemetryEntry.contextTruncationDuration = performance.now() - contextTruncationStartTime
-        getLogger().verbose(`Complete project context processing.`)
+        logger.verbose(`Complete project context processing.`)
         codeScanTelemetryEntry.codewhispererCodeScanSrcPayloadBytes = zipMetadata.srcPayloadSizeInBytes
         codeScanTelemetryEntry.codewhispererCodeScanBuildPayloadBytes = zipMetadata.buildPayloadSizeInBytes
         codeScanTelemetryEntry.codewhispererCodeScanSrcZipFileBytes = zipMetadata.zipFileSizeInBytes
@@ -139,11 +141,8 @@ export async function startSecurityScan(
         const scanName = randomUUID()
         try {
             artifactMap = await getPresignedUrlAndUpload(client, zipMetadata, scope, scanName)
-        } catch (error) {
-            getLogger().error('Failed to upload code artifacts', error)
-            throw error
         } finally {
-            await zipUtil.removeTmpFiles(zipMetadata)
+            await zipUtil.removeTmpFiles(zipMetadata, scope)
             codeScanTelemetryEntry.artifactsUploadDuration = performance.now() - uploadStartTime
         }
 
@@ -162,7 +161,7 @@ export async function startSecurityScan(
         if (scanJob.status === 'Failed') {
             throw new Error(scanJob.errorMessage)
         }
-        getLogger().verbose(`Created security scan job.`)
+        logger.verbose(`Created security scan job.`)
         codeScanTelemetryEntry.codewhispererCodeScanJobId = scanJob.jobId
 
         /**
@@ -178,12 +177,13 @@ export async function startSecurityScan(
          * Step 5: Process and render scan results
          */
         throwIfCancelled(scope)
-        getLogger().verbose(`Security scan job succeeded and start processing result.`)
+        logger.verbose(`Security scan job succeeded and start processing result.`)
         const securityRecommendationCollection = await listScanResults(
             client,
             scanJob.jobId,
             CodeWhispererConstants.codeScanFindingsSchema,
-            projectPath
+            projectPath,
+            scope
         )
         const { total, withFixes } = securityRecommendationCollection.reduce(
             (accumulator, current) => ({
@@ -195,7 +195,7 @@ export async function startSecurityScan(
         codeScanTelemetryEntry.codewhispererCodeScanTotalIssues = total
         codeScanTelemetryEntry.codewhispererCodeScanIssuesWithFixes = withFixes
         throwIfCancelled(scope)
-        getLogger().verbose(`Security scan totally found ${total} issues. ${withFixes} of them have fixes.`)
+        logger.verbose(`Security scan totally found ${total} issues. ${withFixes} of them have fixes.`)
         if (codeScanStartTime > securityScanRender.lastUpdated) {
             showSecurityScanResults(
                 securityPanelViewProvider,
@@ -208,10 +208,10 @@ export async function startSecurityScan(
                 codeScanStartTime
             )
         } else {
-            getLogger().verbose('Received issues from older scan, discarding the results')
+            logger.verbose('Received issues from older scan, discarding the results')
         }
 
-        getLogger().verbose(`Security scan completed.`)
+        logger.verbose(`Security scan completed.`)
     } catch (error) {
         getLogger().error('Security scan failed.', error)
         if (codeScanState.isCancelling()) {

packages/core/src/codewhisperer/service/securityScanHandler.ts

@@ -23,21 +23,24 @@ import {
 import { TelemetryHelper } from '../util/telemetryHelper'
 import request from '../../common/request'
 import { ZipMetadata } from '../util/zipUtil'
+import { getNullLogger } from '../../shared/logger/logger'
 
 export async function listScanResults(
     client: DefaultCodeWhispererClient,
     jobId: string,
     codeScanFindingsSchema: string,
-    projectPath: string
+    projectPath: string,
+    scope: CodeWhispererConstants.CodeAnalysisScope
 ) {
+    const logger = getLoggerForScope(scope)
     const codeScanIssueMap: Map<string, RawCodeScanIssue[]> = new Map()
     const aggregatedCodeScanIssueList: AggregatedCodeScanIssue[] = []
     const requester = (request: codewhispererClient.ListCodeScanFindingsRequest) => client.listCodeScanFindings(request)
     const collection = pageableToCollection(requester, { jobId, codeScanFindingsSchema }, 'nextToken')
     const issues = await collection
         .flatten()
         .map(resp => {
-            getLogger().verbose(`Request id: ${resp.$response.requestId}`)
+            logger.verbose(`Request id: ${resp.$response.requestId}`)
             if ('codeScanFindings' in resp) {
                 return resp.codeScanFindings
             }
@@ -104,7 +107,8 @@ export async function pollScanJobStatus(
     jobId: string,
     scope: CodeWhispererConstants.CodeAnalysisScope
 ) {
-    getLogger().verbose(`Polling scan job status...`)
+    const logger = getLoggerForScope(scope)
+    logger.verbose(`Polling scan job status...`)
     let status: string = 'Pending'
     let timer: number = 0
     while (true) {
@@ -113,11 +117,11 @@ export async function pollScanJobStatus(
             jobId: jobId,
         }
         const resp = await client.getCodeScan(req)
-        getLogger().verbose(`Request id: ${resp.$response.requestId}`)
+        logger.verbose(`Request id: ${resp.$response.requestId}`)
         if (resp.status !== 'Pending') {
             status = resp.status
-            getLogger().verbose(`Scan job status: ${status}`)
-            getLogger().verbose(`Complete Polling scan job status.`)
+            logger.verbose(`Scan job status: ${status}`)
+            logger.verbose(`Complete Polling scan job status.`)
             break
         }
         throwIfCancelled(scope)
@@ -128,8 +132,8 @@ export async function pollScanJobStatus(
                 ? CodeWhispererConstants.codeFileScanJobTimeoutSeconds
                 : CodeWhispererConstants.codeScanJobTimeoutSeconds
         if (timer > timeoutSeconds) {
-            getLogger().verbose(`Scan job status: ${status}`)
-            getLogger().verbose(`Scan job timeout.`)
+            logger.verbose(`Scan job status: ${status}`)
+            logger.verbose(`Scan job timeout.`)
             throw new Error('Scan job timeout.')
         }
     }
@@ -143,7 +147,8 @@ export async function createScanJob(
     scope: CodeWhispererConstants.CodeAnalysisScope,
     scanName: string
 ) {
-    getLogger().verbose(`Creating scan job...`)
+    const logger = getLoggerForScope(scope)
+    logger.verbose(`Creating scan job...`)
     const req: codewhispererClient.CreateCodeScanRequest = {
         artifacts: artifactMap,
         programmingLanguage: {
@@ -156,7 +161,7 @@ export async function createScanJob(
         getLogger().error(`Failed creating scan job. Request id: ${err.requestId}`)
         throw err
     })
-    getLogger().verbose(`Request id: ${resp.$response.requestId}`)
+    logger.verbose(`Request id: ${resp.$response.requestId}`)
     TelemetryHelper.instance.sendCodeScanEvent(languageId, resp.$response.requestId)
     return resp
 }
@@ -167,6 +172,7 @@ export async function getPresignedUrlAndUpload(
     scope: CodeWhispererConstants.CodeAnalysisScope,
     scanName: string
 ) {
+    const logger = getLoggerForScope(scope)
     if (zipMetadata.zipFilePath === '') {
         throw new Error("Zip failure: can't find valid source zip.")
     }
@@ -180,16 +186,16 @@ export async function getPresignedUrlAndUpload(
             },
         },
     }
-    getLogger().verbose(`Prepare for uploading src context...`)
+    logger.verbose(`Prepare for uploading src context...`)
     const srcResp = await client.createUploadUrl(srcReq).catch(err => {
         getLogger().error(`Failed getting presigned url for uploading src context. Request id: ${err.requestId}`)
         throw err
     })
-    getLogger().verbose(`Request id: ${srcResp.$response.requestId}`)
-    getLogger().verbose(`Complete Getting presigned Url for uploading src context.`)
-    getLogger().verbose(`Uploading src context...`)
-    await uploadArtifactToS3(zipMetadata.zipFilePath, srcResp)
-    getLogger().verbose(`Complete uploading src context.`)
+    logger.verbose(`Request id: ${srcResp.$response.requestId}`)
+    logger.verbose(`Complete Getting presigned Url for uploading src context.`)
+    logger.verbose(`Uploading src context...`)
+    await uploadArtifactToS3(zipMetadata.zipFilePath, srcResp, scope)
+    logger.verbose(`Complete uploading src context.`)
     const artifactMap: ArtifactMap = {
         SourceCode: srcResp.uploadId,
     }
@@ -226,7 +232,12 @@ export function throwIfCancelled(scope: CodeWhispererConstants.CodeAnalysisScope
     }
 }
 
-export async function uploadArtifactToS3(fileName: string, resp: CreateUploadUrlResponse) {
+export async function uploadArtifactToS3(
+    fileName: string,
+    resp: CreateUploadUrlResponse,
+    scope: CodeWhispererConstants.CodeAnalysisScope
+) {
+    const logger = getLoggerForScope(scope)
     const encryptionContext = `{"uploadId":"${resp.uploadId}"}`
     const headersObj: Record<string, string> = {
         'Content-MD5': getMd5(fileName),
@@ -244,7 +255,7 @@ export async function uploadArtifactToS3(fileName: string, resp: CreateUploadUrl
             body: readFileSync(fileName),
             headers: resp?.requestHeaders ?? headersObj,
         }).response
-        getLogger().debug(`StatusCode: ${response.status}, Text: ${response.statusText}`)
+        logger.debug(`StatusCode: ${response.status}, Text: ${response.statusText}`)
     } catch (error) {
         getLogger().error(
             `Amazon Q is unable to upload workspace artifacts to Amazon S3 for security scans. For more information, see the Amazon Q documentation or contact your network or organization administrator.`
@@ -254,3 +265,7 @@ export async function uploadArtifactToS3(fileName: string, resp: CreateUploadUrl
         )
     }
 }
+
+export function getLoggerForScope(scope: CodeWhispererConstants.CodeAnalysisScope) {
+    return scope === CodeWhispererConstants.CodeAnalysisScope.FILE ? getNullLogger() : getLogger()
+}

packages/core/src/codewhisperer/util/zipUtil.ts

@@ -11,6 +11,7 @@ import * as CodeWhispererConstants from '../models/constants'
 import { ToolkitError } from '../../shared/errors'
 import { fsCommon } from '../../srcShared/fs'
 import { collectFiles } from '../../amazonqFeatureDev/util/files'
+import { getLoggerForScope } from '../service/securityScanHandler'
 
 export interface ZipMetadata {
     rootDir: string
@@ -182,7 +183,7 @@ export class ZipUtil {
                 throw new ToolkitError(`Unknown code analysis scope: ${scope}`)
             }
 
-            getLogger().debug(`Picked source files: [${[...this._pickedSourceFiles].join(', ')}]`)
+            getLoggerForScope(scope).debug(`Picked source files: [${[...this._pickedSourceFiles].join(', ')}]`)
             const zipFileSize = (await fsCommon.stat(zipFilePath)).size
             return {
                 rootDir: zipDirPath,
@@ -199,10 +200,11 @@ export class ZipUtil {
         }
     }
 
-    public async removeTmpFiles(zipMetadata: ZipMetadata) {
-        getLogger().verbose(`Cleaning up temporary files...`)
+    public async removeTmpFiles(zipMetadata: ZipMetadata, scope: CodeWhispererConstants.CodeAnalysisScope) {
+        const logger = getLoggerForScope(scope)
+        logger.verbose(`Cleaning up temporary files...`)
         await fsCommon.delete(zipMetadata.zipFilePath)
         await fsCommon.delete(zipMetadata.rootDir)
-        getLogger().verbose(`Complete cleaning up temporary files.`)
+        logger.verbose(`Complete cleaning up temporary files.`)
     }
 }

packages/core/src/testE2E/codewhisperer/securityScan.test.ts

@@ -100,7 +100,7 @@ describe('CodeWhisperer security scan', async function () {
         try {
             artifactMap = await getPresignedUrlAndUpload(client, zipMetadata, scope, codeScanName)
         } finally {
-            await zipUtil.removeTmpFiles(zipMetadata)
+            await zipUtil.removeTmpFiles(zipMetadata, scope)
         }
         return {
             artifactMap: artifactMap,
@@ -135,7 +135,8 @@ describe('CodeWhisperer security scan', async function () {
             client,
             scanJob.jobId,
             CodeWhispererConstants.codeScanFindingsSchema,
-            projectPath
+            projectPath,
+            scope
         )
 
         assert.deepStrictEqual(jobStatus, 'Completed')
@@ -169,7 +170,8 @@ describe('CodeWhisperer security scan', async function () {
             client,
             scanJob.jobId,
             CodeWhispererConstants.codeScanFindingsSchema,
-            projectPath
+            projectPath,
+            scope
         )
 
         assert.deepStrictEqual(jobStatus, 'Completed')