codewhisperer: telemetry for supplemental context strategy #3799

We are already able to analyze acceptance rate on either CrossFile or UTG or an aggregated number. Now we need more granular metrics for UTG `resolve source file by file name` vs. `resolve source file by file content` to help us understanding deeper the effectiveness of UTG, thus adding a new field `codewhispererSupplementalContextStrategyId` field representing the strategies being used for Crossfile and UTG.

* Crossfile
    * OpenTabs + BM25

* UTG
   * Resolve by file name (regex)
   * Resolve by file content  

there are 4 values for now
`Empty`, `ByName`, `ByContent`, `OpenTabs_BM25`

Author: Will-ShaoHua

package-lock.json

@@ -3583,7 +3583,7 @@
         "report": "nyc report --reporter=html --reporter=json"
     },
     "devDependencies": {
-        "@aws-toolkits/telemetry": "^1.0.136",
+        "@aws-toolkits/telemetry": "^1.0.143",
         "@cspotcode/source-map-support": "^0.8.1",
         "@sinonjs/fake-timers": "^10.0.2",
         "@types/adm-zip": "^0.4.34",

package.json

@@ -96,6 +96,7 @@ export async function buildListRecommendationRequest(
                   isProcessTimeout: supplementalContexts.isProcessTimeout,
                   contentsLength: supplementalContexts.contentsLength,
                   latency: supplementalContexts.latency,
+                  strategy: supplementalContexts.strategy,
               }
             : undefined
 
@@ -150,6 +151,7 @@ export async function buildGenerateRecommendationRequest(editor: vscode.TextEdit
             isProcessTimeout: supplementalContexts.isProcessTimeout,
             contentsLength: supplementalContexts.contentsLength,
             latency: supplementalContexts.latency,
+            strategy: supplementalContexts.strategy,
         }
     }
 

src/codewhisperer/util/editorContext.ts

@@ -10,13 +10,15 @@ import { BM25Document, BM25Okapi } from './rankBm25'
 import { ToolkitError } from '../../../shared/errors'
 import { UserGroup, crossFileContextConfig, supplemetalContextFetchingTimeoutMsg } from '../../models/constants'
 import { CancellationError } from '../../../shared/utilities/timeoutUtils'
-import { CodeWhispererSupplementalContextItem } from './supplementalContextUtil'
+import { CodeWhispererSupplementalContext, CodeWhispererSupplementalContextItem } from './supplementalContextUtil'
 import { CodeWhispererUserGroupSettings } from '../userGroupUtil'
 import { isTestFile } from './codeParsingUtil'
 import { getFileDistance } from '../../../shared/filesystemUtilities'
 import { getOpenFilesInWindow } from '../../../shared/utilities/editorUtilities'
 import { getLogger } from '../../../shared/logger/logger'
 
+export type CrossFileStrategy = 'OpenTabs_BM25'
+
 type CrossFileSupportedLanguage =
     | 'java'
     | 'python'
@@ -51,14 +53,19 @@ interface Chunk {
 export async function fetchSupplementalContextForSrc(
     editor: vscode.TextEditor,
     cancellationToken: vscode.CancellationToken
-): Promise<CodeWhispererSupplementalContextItem[] | undefined> {
+): Promise<Pick<CodeWhispererSupplementalContext, 'supplementalContextItems' | 'strategy'> | undefined> {
     const shouldProceed = shouldFetchCrossFileContext(
         editor.document.languageId,
         CodeWhispererUserGroupSettings.instance.userGroup
     )
 
     if (!shouldProceed) {
-        return shouldProceed === undefined ? undefined : []
+        return shouldProceed === undefined
+            ? undefined
+            : {
+                  supplementalContextItems: [],
+                  strategy: 'Empty',
+              }
     }
 
     const codeChunksCalculated = crossFileContextConfig.numberOfChunkToFetch
@@ -104,7 +111,10 @@ export async function fetchSupplementalContextForSrc(
 
     // DO NOT send code chunk with empty content
     getLogger().debug(`CodeWhisperer finished fetching crossfile context out of ${relevantCrossFilePaths.length} files`)
-    return supplementalContexts.filter(item => item.content.trim().length !== 0)
+    return {
+        supplementalContextItems: supplementalContexts.filter(item => item.content.trim().length !== 0),
+        strategy: 'OpenTabs_BM25',
+    }
 }
 
 function findBestKChunkMatches(chunkInput: Chunk, chunkReferences: Chunk[], k: number): Chunk[] {

src/codewhisperer/util/supplementalContext/crossFileContextUtil.ts

@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { fetchSupplementalContextForTest } from './utgUtils'
-import { fetchSupplementalContextForSrc } from './crossFileContextUtil'
+import { UtgStrategy, fetchSupplementalContextForTest } from './utgUtils'
+import { CrossFileStrategy, fetchSupplementalContextForSrc } from './crossFileContextUtil'
 import { isTestFile } from './codeParsingUtil'
 import { DependencyGraphFactory } from '../dependencyGraph/dependencyGraphFactory'
 import * as vscode from 'vscode'
@@ -14,12 +14,15 @@ import { getLogger } from '../../../shared/logger/logger'
 
 const performance = globalThis.performance ?? require('perf_hooks').performance
 
+export type SupplementalContextStrategy = CrossFileStrategy | UtgStrategy | 'Empty'
+
 export interface CodeWhispererSupplementalContext {
     isUtg: boolean
     isProcessTimeout: boolean
     supplementalContextItems: CodeWhispererSupplementalContextItem[]
     contentsLength: number
     latency: number
+    strategy: SupplementalContextStrategy
 }
 
 export interface CodeWhispererSupplementalContextItem {
@@ -41,7 +44,9 @@ export async function fetchSupplementalContext(
         fileContent: editor.document.getText(),
     })
 
-    let supplementalContextPromise: Promise<CodeWhispererSupplementalContextItem[] | undefined>
+    let supplementalContextPromise: Promise<
+        Pick<CodeWhispererSupplementalContext, 'supplementalContextItems' | 'strategy'> | undefined
+    >
 
     if (isUtg) {
         supplementalContextPromise = fetchSupplementalContextForTest(editor, cancellationToken)
@@ -55,9 +60,10 @@ export async function fetchSupplementalContext(
                 return {
                     isUtg: isUtg,
                     isProcessTimeout: false,
-                    supplementalContextItems: value,
-                    contentsLength: value.reduce((acc, curr) => acc + curr.content.length, 0),
+                    supplementalContextItems: value.supplementalContextItems,
+                    contentsLength: value.supplementalContextItems.reduce((acc, curr) => acc + curr.content.length, 0),
                     latency: performance.now() - timesBeforeFetching,
+                    strategy: value.strategy,
                 }
             } else {
                 return undefined
@@ -71,6 +77,7 @@ export async function fetchSupplementalContext(
                     supplementalContextItems: [],
                     contentsLength: 0,
                     latency: performance.now() - timesBeforeFetching,
+                    strategy: 'Empty',
                 }
             } else {
                 getLogger().error(

src/codewhisperer/util/supplementalContext/supplementalContextUtil.ts

@@ -18,7 +18,7 @@ import {
 import { ToolkitError } from '../../../shared/errors'
 import { supplemetalContextFetchingTimeoutMsg } from '../../models/constants'
 import { CancellationError } from '../../../shared/utilities/timeoutUtils'
-import { CodeWhispererSupplementalContextItem } from './supplementalContextUtil'
+import { CodeWhispererSupplementalContext, CodeWhispererSupplementalContextItem } from './supplementalContextUtil'
 import { utgConfig } from '../../models/constants'
 import { CodeWhispererUserGroupSettings } from '../userGroupUtil'
 import { UserGroup } from '../../models/constants'
@@ -27,6 +27,8 @@ import { getLogger } from '../../../shared/logger/logger'
 
 type UtgSupportedLanguage = keyof typeof utgLanguageConfigs
 
+export type UtgStrategy = 'ByName' | 'ByContent'
+
 function isUtgSupportedLanguage(languageId: vscode.TextDocument['languageId']): languageId is UtgSupportedLanguage {
     return languageId in utgLanguageConfigs
 }
@@ -58,14 +60,14 @@ export function shouldFetchUtgContext(
 export async function fetchSupplementalContextForTest(
     editor: vscode.TextEditor,
     cancellationToken: vscode.CancellationToken
-): Promise<CodeWhispererSupplementalContextItem[] | undefined> {
+): Promise<Pick<CodeWhispererSupplementalContext, 'supplementalContextItems' | 'strategy'> | undefined> {
     const shouldProceed = shouldFetchUtgContext(
         editor.document.languageId,
         CodeWhispererUserGroupSettings.instance.userGroup
     )
 
     if (!shouldProceed) {
-        return shouldProceed === undefined ? undefined : []
+        return shouldProceed === undefined ? undefined : { supplementalContextItems: [], strategy: 'Empty' }
     }
 
     const languageConfig = utgLanguageConfigs[editor.document.languageId]
@@ -77,24 +79,42 @@ export async function fetchSupplementalContextForTest(
     if (crossSourceFile) {
         // TODO (Metrics): 2. Success count for fetchSourceFileByName (find source file by name)
         getLogger().debug(`CodeWhisperer finished fetching utg context by file name`)
-        return generateSupplementalContextFromFocalFile(crossSourceFile, cancellationToken)
+        return {
+            supplementalContextItems: generateSupplementalContextFromFocalFile(
+                crossSourceFile,
+                'ByName',
+                cancellationToken
+            ),
+            strategy: 'ByName',
+        }
     }
     throwIfCancelled(cancellationToken)
 
     crossSourceFile = await findSourceFileByContent(editor, languageConfig, cancellationToken)
     if (crossSourceFile) {
         // TODO (Metrics): 3. Success count for fetchSourceFileByContent (find source file by content)
         getLogger().debug(`CodeWhisperer finished fetching utg context by file content`)
-        return generateSupplementalContextFromFocalFile(crossSourceFile, cancellationToken)
+        return {
+            supplementalContextItems: generateSupplementalContextFromFocalFile(
+                crossSourceFile,
+                'ByContent',
+                cancellationToken
+            ),
+            strategy: 'ByContent',
+        }
     }
 
     // TODO (Metrics): 4. Failure count - when unable to find focal file (supplemental context empty)
     getLogger().debug(`CodeWhisperer failed to fetch utg context`)
-    return []
+    return {
+        supplementalContextItems: [],
+        strategy: 'Empty',
+    }
 }
 
 function generateSupplementalContextFromFocalFile(
     filePath: string,
+    strategy: UtgStrategy,
     cancellationToken: vscode.CancellationToken
 ): CodeWhispererSupplementalContextItem[] {
     const fileContent = fs.readFileSync(vscode.Uri.file(filePath!).fsPath, 'utf-8')

src/codewhisperer/util/supplementalContext/utgUtils.ts

@@ -299,6 +299,8 @@ export class TelemetryHelper {
             codewhispererSupplementalContextTimeout: supplementalContextMetadata?.isProcessTimeout,
             codewhispererSupplementalContextIsUtg: supplementalContextMetadata?.isUtg,
             codewhispererSupplementalContextLength: supplementalContextMetadata?.contentsLength,
+            // eslint-disable-next-line id-length
+            codewhispererSupplementalContextStrategyId: supplementalContextMetadata?.strategy,
         }
         telemetry.codewhisperer_userTriggerDecision.emit(aggregated)
         this.prevTriggerDecision = this.getAggregatedUserDecision(this.sessionDecisions)

src/codewhisperer/util/telemetryHelper.ts

@@ -133,6 +133,7 @@ describe('recommendationHandler', function () {
                 supplementalContextItems: [],
                 contentsLength: 100,
                 latency: 0,
+                strategy: 'Empty',
             })
             sinon.stub(performance, 'now').returns(0.0)
             handler.startPos = new vscode.Position(1, 0)

src/test/codewhisperer/service/recommendationHandler.test.ts

@@ -46,11 +46,11 @@ describe('crossFileContextUtil', function () {
                 })
                 const actual = await crossFile.fetchSupplementalContextForSrc(myCurrentEditor, fakeCancellationToken)
                 assert.ok(actual)
-                assert.ok(actual.length === 3)
+                assert.ok(actual.supplementalContextItems.length === 3)
 
-                assert.strictEqual(actual[0].content.split('\n').length, 10)
-                assert.strictEqual(actual[1].content.split('\n').length, 10)
-                assert.strictEqual(actual[2].content.split('\n').length, 10)
+                assert.strictEqual(actual.supplementalContextItems[0].content.split('\n').length, 10)
+                assert.strictEqual(actual.supplementalContextItems[1].content.split('\n').length, 10)
+                assert.strictEqual(actual.supplementalContextItems[2].content.split('\n').length, 10)
             }
 
             it('control group', async function () {
@@ -169,7 +169,7 @@ describe('crossFileContextUtil', function () {
 
                 const actual = await crossFile.fetchSupplementalContextForSrc(editor, fakeCancellationToken)
 
-                assert.ok(actual?.length !== undefined && actual.length === 0)
+                assert.ok(actual && actual.supplementalContextItems.length === 0)
             })
         })
     })
@@ -201,7 +201,7 @@ describe('crossFileContextUtil', function () {
 
                 const actual = await crossFile.fetchSupplementalContextForSrc(editor, fakeCancellationToken)
 
-                assert.ok(actual?.length !== undefined && actual.length !== 0)
+                assert.ok(actual && actual.supplementalContextItems.length !== 0)
             })
         })
     })
@@ -233,7 +233,7 @@ describe('crossFileContextUtil', function () {
 
                 const actual = await crossFile.fetchSupplementalContextForSrc(editor, fakeCancellationToken)
 
-                assert.ok(actual?.length !== undefined && actual.length !== 0)
+                assert.ok(actual && actual.supplementalContextItems.length !== 0)
             })
         })
     })