feat(CodeWhisperer) add sending serviceInvocationCount field in codeCoverageTracker (#2895)

Will-ShaoHua · web-flow · commit c3880f0ff5a6 · 2022-09-22T14:26:22.000-07:00
Tracker will increment tracker.serviceInvocationCount by 1 if there is a successful service invocation with specific programming language, i.e. if cwspr get invoked successfully with python, only python tracker will increment serviceInvocationCount.
diff --git a/src/codewhisperer/activation.ts b/src/codewhisperer/activation.ts
@@ -337,10 +337,9 @@ export async function activate(context: ExtContext): Promise<void> {
                     }
                 }
 
-                CodeWhispererCodeCoverageTracker.getTracker(
-                    e.document.languageId,
-                    context.extensionContext.globalState
-                )?.countTotalTokens(e)
+                const codeCoverageTracker = CodeWhispererCodeCoverageTracker.getTracker(e.document.languageId)
+                codeCoverageTracker?.countTotalTokens(e)
+
                 /**
                  * Handle this keystroke event only when
                  * 1. It is in current active editor with cwspr supported file types
@@ -386,8 +385,7 @@ export async function activate(context: ExtContext): Promise<void> {
                 await InlineCompletion.instance.rejectRecommendation(vscode.window.activeTextEditor)
                 if (vscode.window.activeTextEditor) {
                     CodeWhispererCodeCoverageTracker.getTracker(
-                        vscode.window.activeTextEditor.document.languageId,
-                        context.extensionContext.globalState
+                        vscode.window.activeTextEditor.document.languageId
                     )?.updateAcceptedTokensCount(vscode.window.activeTextEditor)
                 }
             }),
@@ -468,10 +466,7 @@ export async function activate(context: ExtContext): Promise<void> {
                     }
                 }
 
-                CodeWhispererCodeCoverageTracker.getTracker(
-                    e.document.languageId,
-                    context.extensionContext.globalState
-                )?.countTotalTokens(e)
+                CodeWhispererCodeCoverageTracker.getTracker(e.document.languageId)?.countTotalTokens(e)
 
                 if (
                     e.document === vscode.window.activeTextEditor?.document &&
diff --git a/src/codewhisperer/commands/onAcceptance.ts b/src/codewhisperer/commands/onAcceptance.ts
@@ -97,7 +97,7 @@ export async function onAcceptance(acceptanceEntry: OnRecommendationAcceptanceEn
             language: languageContext.language,
         })
         const codeRangeAfterFormat = new vscode.Range(start, acceptanceEntry.editor.selection.active)
-        CodeWhispererCodeCoverageTracker.getTracker(languageContext.language, globalStorage)?.countAcceptedTokens(
+        CodeWhispererCodeCoverageTracker.getTracker(languageContext.language)?.countAcceptedTokens(
             codeRangeAfterFormat,
             acceptanceEntry.editor.document.getText(codeRangeAfterFormat),
             acceptanceEntry.editor.document.fileName
diff --git a/src/codewhisperer/service/recommendationHandler.ts b/src/codewhisperer/service/recommendationHandler.ts
@@ -24,6 +24,7 @@ import {
     CodewhispererTriggerType,
     Result,
 } from '../../shared/telemetry/telemetry'
+import { CodeWhispererCodeCoverageTracker } from '../tracker/codewhispererCodeCoverageTracker'
 
 /**
  * This class is for getRecommendation/listRecommendation API calls and its states
@@ -248,6 +249,9 @@ export class RecommendationHandler {
                     reason: reason ? reason.substring(0, 200) : undefined,
                 })
             }
+            if (invocationResult == 'Succeeded') {
+                CodeWhispererCodeCoverageTracker.getTracker(languageContext.language)?.incrementServiceInvocationCount()
+            }
             if (config.isIncludeSuggestionsWithCodeReferencesEnabled === false) {
                 recommendation.forEach((r, index) => {
                     if (r.references !== undefined && r.references.length) {
diff --git a/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts b/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts
@@ -11,6 +11,7 @@ import { vsCodeState } from '../models/model'
 import { distance } from 'fastest-levenshtein'
 import { CodewhispererLanguage, telemetry } from '../../shared/telemetry/telemetry'
 import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
+import { TelemetryHelper } from '../util/telemetryHelper'
 
 interface CodeWhispererToken {
     range: vscode.Range
@@ -27,32 +28,46 @@ export class CodeWhispererCodeCoverageTracker {
     private _timer?: NodeJS.Timer
     private _startTime: number
     private _language: CodewhispererLanguage
+    private _serviceInvocationCount: number
 
-    private constructor(language: CodewhispererLanguage, private readonly _globals: vscode.Memento) {
+    private constructor(language: CodewhispererLanguage, private readonly globals: vscode.Memento) {
         this._acceptedTokens = {}
         this._totalTokens = {}
         this._startTime = 0
         this._language = language
+        this._serviceInvocationCount = 0
+    }
+
+    public get serviceInvocationCount(): number {
+        return this._serviceInvocationCount
     }
 
     public get acceptedTokens(): { [key: string]: CodeWhispererToken[] } {
         return this._acceptedTokens
     }
+
     public get totalTokens(): { [key: string]: number } {
         return this._totalTokens
     }
 
+    public isActive(): boolean {
+        const isTermsAccepted = this.globals.get<boolean>(CodeWhispererConstants.termsAcceptedKey) || false
+        return TelemetryHelper.instance.isTelemetryEnabled() && isTermsAccepted
+    }
+
     public countAcceptedTokens(range: vscode.Range, text: string, filename: string) {
-        const terms = this._globals.get<boolean>(CodeWhispererConstants.termsAcceptedKey) || false
-        if (!terms) return
+        if (!this.isActive()) return
         // generate accepted recommendation token and stored in collection
         this.addAcceptedTokens(filename, { range: range, text: text, accepted: text.length })
         this.addTotalTokens(filename, text.length)
     }
 
+    public incrementServiceInvocationCount() {
+        this._serviceInvocationCount += 1
+    }
+
     public flush() {
-        const terms = this._globals.get<boolean>(CodeWhispererConstants.termsAcceptedKey) || false
-        if (!terms) {
+        if (!this.isActive()) {
             this._totalTokens = {}
             this._acceptedTokens = {}
             this.closeTimer()
@@ -107,7 +122,7 @@ export class CodeWhispererCodeCoverageTracker {
             codewhispererLanguage: this._language,
             codewhispererAcceptedTokens: acceptedTokens,
             codewhispererPercentage: percentage ? percentage : 0,
-            successCount: 0,
+            successCount: this._serviceInvocationCount,
         })
     }
 
@@ -136,14 +151,19 @@ export class CodeWhispererCodeCoverageTracker {
             } catch (e) {
                 getLogger().verbose(`Exception Thrown from CodeWhispererCodeCoverageTracker: ${e}`)
             } finally {
-                this._totalTokens = {}
-                this._acceptedTokens = {}
-                this._startTime = 0
+                this.resetTracker()
                 this.closeTimer()
             }
         }, CodeWhispererConstants.defaultCheckPeriodMillis)
     }
 
+    private resetTracker() {
+        this._totalTokens = {}
+        this._acceptedTokens = {}
+        this._startTime = 0
+        this._serviceInvocationCount = 0
+    }
+
     private closeTimer() {
         if (this._timer !== undefined) {
             clearTimeout(this._timer)
@@ -190,12 +210,20 @@ export class CodeWhispererCodeCoverageTracker {
     }
 
     public static readonly instances = new Map<string, CodeWhispererCodeCoverageTracker>()
-    public static getTracker(language: string, memento: vscode.Memento): CodeWhispererCodeCoverageTracker | undefined {
-        if (runtimeLanguageContext.isLanguageSupported(language)) {
-            const instance = this.instances.get(language) ?? new this(language as CodewhispererLanguage, memento)
-            this.instances.set(language, instance)
-            return instance
+
+    public static getTracker(
+        language: string,
+        memeto: vscode.Memento = globals.context.globalState
+    ): CodeWhispererCodeCoverageTracker | undefined {
+        if (!runtimeLanguageContext.isLanguageSupported(language)) {
+            return undefined
+        }
+        const cwsprLanguage = runtimeLanguageContext.mapVscLanguageToCodeWhispererLanguage(language)
+        if (!cwsprLanguage) {
+            return undefined
         }
-        return undefined
+        const instance = this.instances.get(language) ?? new this(cwsprLanguage, memeto)
+        this.instances.set(language, instance)
+        return instance
     }
 }
diff --git a/src/codewhisperer/util/telemetryHelper.ts b/src/codewhisperer/util/telemetryHelper.ts
@@ -2,6 +2,8 @@
  * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
+import globals from '../../shared/extensionGlobals'
+
 import { runtimeLanguageContext } from './runtimeLanguageContext'
 import { RecommendationsList } from '../client/codewhisperer'
 import { LicenseUtil } from './licenseUtil'
@@ -126,4 +128,8 @@ export class TelemetryHelper {
         }
         return i === acceptIndex ? 'Accept' : 'Ignore'
     }
+
+    public isTelemetryEnabled(): boolean {
+        return globals.telemetry.telemetryEnabled
+    }
 }
diff --git a/src/dynamicResources/model/supported_resources.json b/src/dynamicResources/model/supported_resources.json
@@ -386,7 +386,7 @@
         "documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudfront-keygroup.html"
     },
     "AWS::CloudFront::OriginAccessControl": {
-        "operations": ["CREATE", "DELETE", "LIST", "READ", "UPDATE" ]
+        "operations": ["CREATE", "DELETE", "LIST", "READ", "UPDATE"]
     },
     "AWS::CloudFront::OriginRequestPolicy": {
         "operations": ["CREATE", "DELETE", "LIST", "READ", "UPDATE"],
@@ -1796,10 +1796,10 @@
         "documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lookoutvision-project.html"
     },
     "AWS::M2::Application": {
-        "operations": ["CREATE", "READ", "UPDATE", "DELETE", "LIST" ]
+        "operations": ["CREATE", "READ", "UPDATE", "DELETE", "LIST"]
     },
     "AWS::M2::Environment": {
-        "operations": ["CREATE", "READ", "UPDATE", "DELETE", "LIST" ]
+        "operations": ["CREATE", "READ", "UPDATE", "DELETE", "LIST"]
     },
     "AWS::MSK::BatchScramSecret": {
         "operations": ["CREATE", "DELETE", "READ", "UPDATE"],
diff --git a/src/test/codewhisperer/service/recommendationHandler.test.ts b/src/test/codewhisperer/service/recommendationHandler.test.ts
@@ -14,6 +14,8 @@ import { createMockTextEditor, resetCodeWhispererGlobalVariables } from '../test
 import { TelemetryHelper } from '../../../codewhisperer/util/telemetryHelper'
 import { RecommendationHandler } from '../../../codewhisperer/service/recommendationHandler'
 import { stub } from '../../utilities/stubber'
+import { CodeWhispererCodeCoverageTracker } from '../../../codewhisperer/tracker/codewhispererCodeCoverageTracker'
+import { FakeMemento } from '../../fakeExtensionContext'
 
 const performance = globalThis.performance ?? require('perf_hooks').performance
 
@@ -29,6 +31,7 @@ describe('recommendationHandler', function () {
     })
 
     describe('getRecommendations', async function () {
+        const fakeMemeto = new FakeMemento()
         const mockClient = stub(DefaultCodeWhispererClient)
         const mockEditor = createMockTextEditor()
 
@@ -44,6 +47,13 @@ describe('recommendationHandler', function () {
         })
 
         it('should assign correct recommendations given input', async function () {
+            assert.strictEqual(CodeWhispererCodeCoverageTracker.instances.size, 0)
+            assert.strictEqual(
+                CodeWhispererCodeCoverageTracker.getTracker(mockEditor.document.languageId, fakeMemeto)
+                    ?.serviceInvocationCount,
+                0
+            )
+
             const mockServerResult = {
                 recommendations: [{ content: "print('Hello World!')" }, { content: '' }],
                 $response: {
@@ -61,6 +71,11 @@ describe('recommendationHandler', function () {
             const actual = handler.recommendations
             const expected: RecommendationsList = [{ content: "print('Hello World!')" }, { content: '' }]
             assert.deepStrictEqual(actual, expected)
+            assert.strictEqual(
+                CodeWhispererCodeCoverageTracker.getTracker(mockEditor.document.languageId, fakeMemeto)
+                    ?.serviceInvocationCount,
+                1
+            )
         })
 
         it('should assign request id correctly', async function () {
diff --git a/src/test/codewhisperer/testUtil.ts b/src/test/codewhisperer/testUtil.ts
@@ -9,10 +9,12 @@ import * as codewhispererClient from '../../codewhisperer/client/codewhisperer'
 import { vsCodeState, AcceptedSuggestionEntry } from '../../codewhisperer/models/model'
 import { MockDocument } from '../fake/fakeDocument'
 import { getLogger } from '../../shared/logger'
+import { CodeWhispererCodeCoverageTracker } from '../../codewhisperer/tracker/codewhispererCodeCoverageTracker'
 
 export function resetCodeWhispererGlobalVariables() {
     vsCodeState.isIntelliSenseActive = false
     vsCodeState.isCodeWhispererEditing = false
+    CodeWhispererCodeCoverageTracker.instances.clear()
 }
 
 export function createMockDocument(
diff --git a/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts b/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,8 @@`
`2`	`2`	`* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.`
`3`	`3`	`* SPDX-License-Identifier: Apache-2.0`
`4`	`4`	`*/`
	`5`	`+import globals from '../../shared/extensionGlobals'`
	`6`	`+`
`5`	`7`	`import { runtimeLanguageContext } from './runtimeLanguageContext'`
`6`	`8`	`import { RecommendationsList } from '../client/codewhisperer'`
`7`	`9`	`import { LicenseUtil } from './licenseUtil'`
`@@ -126,4 +128,8 @@ export class TelemetryHelper {`
`126`	`128`	`}`
`127`	`129`	`return i === acceptIndex ? 'Accept' : 'Ignore'`
`128`	`130`	`}`
	`131`	`+`
	`132`	`+ public isTelemetryEnabled(): boolean {`
	`133`	`+ return globals.telemetry.telemetryEnabled`
	`134`	`+ }`
`129`	`135`	`}`