fix(codewhisperer): Improve Code percentage reporting (#4350)

Author: leigaol

.changes/next-release/Bug Fix-2ce4c789-d0e5-425c-affb-6cac6e5069fd.json

@@ -0,0 +1,4 @@
+{
+	"type": "Bug Fix",
+	"description": "CodeWhisperer: Improve CodePercentage telemetry reporting"
+}

src/codewhisperer/client/user-service-2.json

@@ -417,7 +417,8 @@
                 "programmingLanguage": { "shape": "ProgrammingLanguage" },
                 "acceptedCharacterCount": { "shape": "PrimitiveInteger" },
                 "totalCharacterCount": { "shape": "PrimitiveInteger" },
-                "timestamp": { "shape": "Timestamp" }
+                "timestamp": { "shape": "Timestamp" },
+                "unmodifiedAcceptedCharacterCount": { "shape": "PrimitiveInteger" }
             }
         },
         "CodeGenerationId": {

src/codewhisperer/commands/onAcceptance.ts

@@ -66,10 +66,10 @@ export async function onAcceptance(acceptanceEntry: OnRecommendationAcceptanceEn
             completionType: acceptanceEntry.completionType,
             language: languageContext.language,
         })
-        const codeRangeAfterFormat = new vscode.Range(start, acceptanceEntry.editor.selection.active)
+        const insertedCoderange = new vscode.Range(start, end)
         CodeWhispererCodeCoverageTracker.getTracker(languageContext.language)?.countAcceptedTokens(
-            codeRangeAfterFormat,
-            acceptanceEntry.editor.document.getText(codeRangeAfterFormat),
+            insertedCoderange,
+            acceptanceEntry.editor.document.getText(insertedCoderange),
             acceptanceEntry.editor.document.fileName
         )
         if (acceptanceEntry.references !== undefined) {

src/codewhisperer/commands/onInlineAcceptance.ts

@@ -116,10 +116,10 @@ export async function onInlineAcceptance(
             completionType: acceptanceEntry.completionType,
             language: languageContext.language,
         })
-        const codeRangeAfterFormat = new vscode.Range(start, acceptanceEntry.editor.selection.active)
+        const insertedCoderange = new vscode.Range(start, end)
         CodeWhispererCodeCoverageTracker.getTracker(languageContext.language, globalStorage)?.countAcceptedTokens(
-            codeRangeAfterFormat,
-            acceptanceEntry.editor.document.getText(codeRangeAfterFormat),
+            insertedCoderange,
+            acceptanceEntry.editor.document.getText(insertedCoderange),
             acceptanceEntry.editor.document.fileName
         )
         if (acceptanceEntry.references !== undefined) {

src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts

@@ -59,14 +59,6 @@ export class CodeWhispererCodeCoverageTracker {
         return TelemetryHelper.instance.isTelemetryEnabled() && AuthUtil.instance.isConnected()
     }
 
-    public countAcceptedTokens(range: vscode.Range, text: string, filename: string) {
-        if (!this.isActive()) {
-            return
-        }
-        // generate accepted recommendation token and stored in collection
-        this.addAcceptedTokens(filename, { range: range, text: text, accepted: text.length })
-    }
-
     public incrementServiceInvocationCount() {
         this._serviceInvocationCount += 1
     }
@@ -85,6 +77,8 @@ export class CodeWhispererCodeCoverageTracker {
         }
     }
 
+    // TODO: Improve the range tracking of the accepted recommendation
+    // TODO: use the editor of the filename, not the current editor
     public updateAcceptedTokensCount(editor: vscode.TextEditor) {
         const filename = editor.document.fileName
         if (filename in this._acceptedTokens) {
@@ -112,17 +106,25 @@ export class CodeWhispererCodeCoverageTracker {
         if (vscode.window.activeTextEditor) {
             this.updateAcceptedTokensCount(vscode.window.activeTextEditor)
         }
+        // the accepted characters without counting user modification
         let acceptedTokens = 0
+        // the accepted characters after calculating user modificaiton
+        let unmodifiedAcceptedTokens = 0
         for (const filename in this._acceptedTokens) {
             this._acceptedTokens[filename].forEach(v => {
                 if (filename in this._totalTokens && this._totalTokens[filename] >= v.accepted) {
-                    acceptedTokens += v.accepted
+                    unmodifiedAcceptedTokens += v.accepted
+                    acceptedTokens += v.text.length
                 }
             })
         }
         const percentCount = ((acceptedTokens / totalTokens) * 100).toFixed(2)
         const percentage = Math.round(parseInt(percentCount))
         const selectedCustomization = getSelectedCustomization()
+        if (this._serviceInvocationCount <= 0) {
+            getLogger().debug(`Skip emiting code contribution metric`)
+            return
+        }
         telemetry.codewhisperer_codePercentage.emit({
             codewhispererTotalTokens: totalTokens,
             codewhispererLanguage: this._language,
@@ -142,6 +144,7 @@ export class CodeWhispererCodeCoverageTracker {
                             languageName: runtimeLanguageContext.toRuntimeLanguage(this._language),
                         },
                         acceptedCharacterCount: acceptedTokens,
+                        unmodifiedAcceptedCharacterCount: unmodifiedAcceptedTokens,
                         totalCharacterCount: totalTokens,
                         timestamp: new Date(Date.now()),
                     },
@@ -226,15 +229,24 @@ export class CodeWhispererCodeCoverageTracker {
         }
     }
 
+    public countAcceptedTokens(range: vscode.Range, text: string, filename: string) {
+        if (!this.isActive()) {
+            return
+        }
+        // generate accepted recommendation token and stored in collection
+        this.addAcceptedTokens(filename, { range: range, text: text, accepted: text.length })
+        this.addTotalTokens(filename, text.length)
+    }
+
     public countTotalTokens(e: vscode.TextDocumentChangeEvent) {
         // ignore no contentChanges. ignore contentChanges from other plugins (formatters)
-        // only include contentChanges from user action.
+        // only include contentChanges from user keystroke input(one character input).
         // Also ignore deletion events due to a known issue of tracking deleted CodeWhiperer tokens.
         if (
             !runtimeLanguageContext.isLanguageSupported(e.document.languageId) ||
             vsCodeState.isCodeWhispererEditing ||
             e.contentChanges.length !== 1 ||
-            e.contentChanges[0].text.length === 0
+            e.contentChanges[0].text.length !== 1
         ) {
             return
         }

src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts

@@ -206,6 +206,14 @@ describe('codewhispererCodecoverageTracker', function () {
                 accepted: 1,
             })
         })
+        it('Should increase TotalTokens', function () {
+            if (!tracker) {
+                assert.fail()
+            }
+            tracker.countAcceptedTokens(new vscode.Range(0, 0, 0, 1), 'a', 'test.py')
+            tracker.countAcceptedTokens(new vscode.Range(0, 0, 0, 1), 'b', 'test.py')
+            assert.deepStrictEqual(tracker.totalTokens['test.py'], 2)
+        })
     })
 
     describe('countTotalTokens', function () {
@@ -224,7 +232,7 @@ describe('codewhispererCodecoverageTracker', function () {
             CodeWhispererCodeCoverageTracker.instances.clear()
         })
 
-        it('Should skip when user copy large files', function () {
+        it('Should skip when content change size is not 1', function () {
             if (!tracker) {
                 assert.fail()
             }
@@ -266,7 +274,7 @@ describe('codewhispererCodecoverageTracker', function () {
             assert.ok(!startedSpy.called)
         })
 
-        it('Should reduce tokens when delete', function () {
+        it('Should not reduce tokens when delete', function () {
             if (!tracker) {
                 assert.fail()
             }
@@ -276,14 +284,26 @@ describe('codewhispererCodecoverageTracker', function () {
                 document: doc,
                 contentChanges: [
                     {
-                        range: new vscode.Range(0, 0, 0, 3),
+                        range: new vscode.Range(0, 0, 0, 1),
                         rangeOffset: 0,
                         rangeLength: 0,
-                        text: 'aaa',
+                        text: 'a',
                     },
                 ],
             })
-            assert.strictEqual(tracker?.totalTokens[doc.fileName], 3)
+            tracker.countTotalTokens({
+                reason: undefined,
+                document: doc,
+                contentChanges: [
+                    {
+                        range: new vscode.Range(0, 0, 0, 1),
+                        rangeOffset: 0,
+                        rangeLength: 0,
+                        text: 'b',
+                    },
+                ],
+            })
+            assert.strictEqual(tracker?.totalTokens[doc.fileName], 2)
             tracker.countTotalTokens({
                 reason: undefined,
                 document: doc,
@@ -296,7 +316,7 @@ describe('codewhispererCodecoverageTracker', function () {
                     },
                 ],
             })
-            assert.strictEqual(tracker?.totalTokens[doc.fileName], 3)
+            assert.strictEqual(tracker?.totalTokens[doc.fileName], 2)
         })
 
         it('Should add tokens when type', function () {
@@ -381,21 +401,21 @@ describe('codewhispererCodecoverageTracker', function () {
             const tracker = CodeWhispererCodeCoverageTracker.getTracker(language)
 
             const assertTelemetry = assertTelemetryCurried('codewhisperer_codePercentage')
+            tracker?.incrementServiceInvocationCount()
             tracker?.addAcceptedTokens(`test.py`, { range: new vscode.Range(0, 0, 0, 7), text: `print()`, accepted: 7 })
             tracker?.addTotalTokens(`test.py`, 100)
             tracker?.emitCodeWhispererCodeContribution()
-
             assertTelemetry({
                 codewhispererTotalTokens: 100,
                 codewhispererLanguage: language,
                 codewhispererAcceptedTokens: 7,
                 codewhispererPercentage: 7,
-                successCount: 0,
+                successCount: 1,
                 codewhispererUserGroup: 'Control',
             })
         })
 
-        it('should emit correct code coverage telemetry in java file', async function () {
+        it('should emit correct code coverage telemetry when success count = 0', async function () {
             await globals.context.globalState.update(CodeWhispererConstants.userGroupKey, {
                 group: CodeWhispererConstants.UserGroup.Control,
                 version: extensionVersion,
@@ -409,14 +429,16 @@ describe('codewhispererCodecoverageTracker', function () {
                 text: `public static main`,
                 accepted: 18,
             })
+            tracker?.incrementServiceInvocationCount()
+            tracker?.incrementServiceInvocationCount()
             tracker?.addTotalTokens(`test.java`, 30)
             tracker?.emitCodeWhispererCodeContribution()
             assertTelemetry({
                 codewhispererTotalTokens: 30,
                 codewhispererLanguage: 'java',
                 codewhispererAcceptedTokens: 18,
                 codewhispererPercentage: 60,
-                successCount: 0,
+                successCount: 2,
                 codewhispererUserGroup: 'Control',
             })
         })