fix(cwspr): include copied code in codePercentage metrics (#4177)

Author: leigaol

.changes/next-release/bugfix-ff1f68bc-fd00-4a21-864d-11d76b740dd3.json

@@ -0,0 +1,4 @@
+{
+  "type" : "bugfix",
+  "description" : "CodeWhisperer: Include copied code in percentage code written metrics"
+}

plugins/toolkit/jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt

@@ -89,6 +89,10 @@ abstract class CodeWhispererCodeCoverageTracker(
                     runReadAction {
                         // also increment total tokens because accepted tokens are part of it
                         incrementTotalTokens(rangeMarker.document, originalRecommendation.length)
+                        // avoid counting CodeWhisperer inserted suggestion twice in total tokens
+                        if (rangeMarker.textRange.length in 2..49 && originalRecommendation.trim().isNotEmpty()) {
+                            incrementTotalTokens(rangeMarker.document, -rangeMarker.textRange.length)
+                        }
                     }
                 }
             }
@@ -118,14 +122,18 @@ abstract class CodeWhispererCodeCoverageTracker(
             if (event.oldTimeStamp == 0L) return
         }
         // only count total tokens when it is a user keystroke input
-        // do not count doc changes from copy & paste of >=2 characters
+        // do not count doc changes from copy & paste of >=50 characters
         // do not count other changes from formatter, git command, etc
         // edge case: event can be from user hit enter with indentation where change is \n\t\t, count as 1 char increase in total chars
         // when event is auto closing [{(', there will be 2 separated events, both count as 1 char increase in total chars
         val text = event.newFragment.toString()
         if ((event.newLength == 1 && event.oldLength == 0) || (text.startsWith('\n') && text.trim().isEmpty())) {
             incrementTotalTokens(event.document, 1)
             return
+        } else if (event.newLength < 50 && text.trim().isNotEmpty()) {
+            // count doc changes from <50 multi character input as total user written code
+            // ignore all white space changes, this usually comes from IntelliJ formatting
+            incrementTotalTokens(event.document, event.newLength)
         }
     }
 

plugins/toolkit/jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt

@@ -245,7 +245,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
     }
 
     @Test
-    fun `test tracker is not listening to multi char input and will not increment totalTokens - add new code`() {
+    fun `test tracker is not listening to multi char input more than 50 and will not increment totalTokens - add new code`() {
         sut = spy(TestCodePercentageTracker(project, TOTAL_SECONDS_IN_MINUTE, CodeWhispererPython.INSTANCE))
         CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut
         sut.activateTrackerIfNotActive()
@@ -259,15 +259,15 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
         val captor = argumentCaptor<DocumentEvent>()
         verify(sut, Times(1)).documentChanged(captor.capture())
         assertThat(captor.firstValue.newFragment.toString()).isEqualTo(pythonTestLeftContext)
-        assertThat(sut.totalTokensSize).isEqualTo(0)
+        assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length)
 
-        val anotherCode = "(x, y):"
+        val anotherCode = "(x, y):".repeat(8)
         runInEdtAndWait {
             WriteCommandAction.runWriteCommandAction(project) {
                 fixture.editor.appendString(anotherCode)
             }
         }
-        assertThat(sut.totalTokensSize).isEqualTo(0)
+        assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length)
     }
 
     @Test