fix(cwspr): update codePercentage metrics threshold from 500 to 50 #4522

Problem:
By conducting one week of data collection, we find that multi character text change which has more than 50 characters only account for 2% of total document change, but they somehow contributed to more than 98% of characters in the doc. This is because user rarely copy and paste code into their editor, but when they do, the copy will result in huge number of character increase. This kind of increase is not what we want to capture in the cwspr percentage code written metric.

50 is a good threshold that includes copy of small code snippets and IDE intelliSense accepted code.

Solution:
Ignore any multi character input more than 50 characters.
The userDetails JSON will be removed in the future. We want to monitor it for a few weeks longer.

Author: leigaol

.changes/next-release/Bug Fix-0ac3c863-5f66-450b-a0f5-f1f43a07ec76.json

@@ -0,0 +1,4 @@
+{
+	"type": "Bug Fix",
+	"description": "CodeWhisperer: Include copied code in percentage code written metrics"
+}

packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts

@@ -338,16 +338,17 @@ export class CodeWhispererCodeCoverageTracker {
             this._userInputDetails.lt1.count += 1
             this._userInputDetails.lt1.total += characterIncrease
         }
-        // also include multi character input within 500 characters (not from CWSPR)
+        // also include multi character input within 50 characters (not from CWSPR)
         else if (
             e.contentChanges.length === 1 &&
             e.contentChanges[0].text.length > 1 &&
             TelemetryHelper.instance.lastSuggestionInDisplay !== e.contentChanges[0].text
         ) {
             const multiCharInputSize = e.contentChanges[0].text.length
 
-            // select 500 as the cut-off threshold for counting user input.
-            if (multiCharInputSize < 500) {
+            // select 50 as the cut-off threshold for counting user input.
+            // ignore all white space multi char input, this usually comes from reformat.
+            if (multiCharInputSize < 50 && e.contentChanges[0].text.trim().length > 0) {
                 this.addTotalTokens(e.document.fileName, multiCharInputSize)
             }
 

packages/core/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts

@@ -232,7 +232,7 @@ describe('codewhispererCodecoverageTracker', function () {
             CodeWhispererCodeCoverageTracker.instances.clear()
         })
 
-        it('Should skip when content change size is more than 500', function () {
+        it('Should skip when content change size is more than 50', function () {
             if (!tracker) {
                 assert.fail()
             }
@@ -251,7 +251,7 @@ describe('codewhispererCodecoverageTracker', function () {
             assert.strictEqual(Object.keys(tracker.totalTokens).length, 0)
         })
 
-        it('Should not skip when content change size is less than 500', function () {
+        it('Should not skip when content change size is less than 50', function () {
             if (!tracker) {
                 assert.fail()
             }
@@ -260,15 +260,15 @@ describe('codewhispererCodecoverageTracker', function () {
                 document: createMockDocument(),
                 contentChanges: [
                     {
-                        range: new vscode.Range(0, 0, 0, 300),
+                        range: new vscode.Range(0, 0, 0, 49),
                         rangeOffset: 0,
-                        rangeLength: 300,
-                        text: 'def twoSum(nums, target): for '.repeat(10),
+                        rangeLength: 49,
+                        text: 'a = 123'.repeat(7),
                     },
                 ],
             })
             assert.strictEqual(Object.keys(tracker.totalTokens).length, 1)
-            assert.strictEqual(Object.values(tracker.totalTokens)[0], 300)
+            assert.strictEqual(Object.values(tracker.totalTokens)[0], 49)
         })
 
         it('Should skip when CodeWhisperer is editing', function () {