CodeWhisperer: Make CodeWhisperer CodePercentage metric not track deletion events (#3866)

1. This is also to prevent the case when the percentage
can go over 100%
2. Add a TODO for the ideal approach

Author: andrewyuq

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/editor/CodeWhispererEditorListener.kt

@@ -24,6 +24,8 @@ class CodeWhispererEditorListener : EditorFactoryListener {
                 // If language is supported, install document listener for CodeWhisperer service
                 editor.document.addDocumentListener(
                     object : BulkAwareDocumentListener {
+                        // TODO: Track only deletion changes within the current 5-min interval which will give
+                        // the most accurate code percentage data.
                         override fun documentChanged(event: DocumentEvent) {
                             if (!isCodeWhispererEnabled(project)) return
                             CodeWhispererInvocationStatus.getInstance().documentChanged()

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt

@@ -81,10 +81,8 @@ abstract class CodeWhispererCodeCoverageTracker(
                 override fun afterAccept(states: InvocationContext, sessionContext: SessionContext, rangeMarker: RangeMarker) {
                     if (states.requestContext.fileContextInfo.programmingLanguage != language) return
                     rangeMarkers.add(rangeMarker)
-                    val originalRecommendation = extractRangeMarkerString(rangeMarker)
-                    originalRecommendation?.let {
-                        rangeMarker.putUserData(KEY_REMAINING_RECOMMENDATION, it)
-                    }
+                    val originalRecommendation = extractRangeMarkerString(rangeMarker) ?: return
+                    rangeMarker.putUserData(KEY_REMAINING_RECOMMENDATION, originalRecommendation)
                 }
             }
         )
@@ -114,6 +112,9 @@ abstract class CodeWhispererCodeCoverageTracker(
         }
         // This case capture IDE reformatting the document, which will be blank string
         if (isDocumentEventFromReformatting(event)) return
+
+        // Don't capture deletion events
+        if (event.newLength <= event.oldLength) return
         incrementTotalTokens(event.document, event.newLength - event.oldLength)
     }
 
@@ -181,6 +182,8 @@ abstract class CodeWhispererCodeCoverageTracker(
     }
 
     internal fun emitCodeWhispererCodeContribution() {
+        // If the user is inactive, don't emit the telemetry
+        if (percentage == null) return
         rangeMarkers.forEach { rangeMarker ->
             if (!rangeMarker.isValid) return@forEach
             // if users add more code upon the recommendation generated from CodeWhisperer, we consider those added part as userToken but not CwsprTokens

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt

@@ -246,7 +246,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
     }
 
     @Test
-    fun `test tracker is listening to document changes and increment totalTokens - delete code`() {
+    fun `test tracker is listening to document changes and increment totalTokens - delete code should not affect`() {
         val pythonTracker = TestCodePercentageTracker(
             project,
             TOTAL_SECONDS_IN_MINUTE,
@@ -265,7 +265,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
             }
         }
 
-        assertThat(pythonTracker.totalTokensSize).isEqualTo(pythonTestLeftContext.length - 3)
+        assertThat(pythonTracker.totalTokensSize).isEqualTo(pythonTestLeftContext.length)
     }
 
     @Test

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererTelemetryTest.kt

@@ -530,7 +530,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         CodeWhispererCodeCoverageTracker.getInstance(project, CodeWhispererPython.INSTANCE).dispose()
 
         val acceptedTokensSize = pythonResponse.completions()[0].content().length - deletedTokenByUser
-        val totalTokensSize = pythonTestLeftContext.length + pythonResponse.completions()[0].content().length - deletedTokenByUser
+        val totalTokensSize = pythonTestLeftContext.length + pythonResponse.completions()[0].content().length
 
         val metricCaptor = argumentCaptor<MetricEvent>()
         verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
@@ -578,7 +578,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         }
 
         val acceptedTokensSize = pythonResponse.completions()[0].content().length
-        val totalTokensSize = pythonTestLeftContext.length + pythonResponse.completions()[0].content().length + anotherCodeSnippet.length - 1
+        val totalTokensSize = pythonTestLeftContext.length + pythonResponse.completions()[0].content().length + anotherCodeSnippet.length
 
         val metricCaptor = argumentCaptor<MetricEvent>()
         verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())