fix(codewhisperer): Improve Code percentage reporting (#4120)

* update codeCoverage API with new fields

* improve code percentage reporting of CW

Author: leigaol

.changes/next-release/bugfix-9969999c-a86d-4697-b398-976cef5128df.json

@@ -0,0 +1,4 @@
+{
+  "type" : "bugfix",
+  "description" : "CodeWhisperer: Improve CodePercentage telemetry reporting"
+}

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt

@@ -100,7 +100,8 @@ interface CodeWhispererClientAdaptor : Disposable {
         language: CodeWhispererProgrammingLanguage,
         customizationArn: String?,
         acceptedTokenCount: Int,
-        totalTokenCount: Int
+        totalTokenCount: Int,
+        unmodifiedAcceptedTokenCount: Int?
     ): SendTelemetryEventResponse
 
     fun sendUserModificationTelemetry(
@@ -297,7 +298,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         language: CodeWhispererProgrammingLanguage,
         customizationArn: String?,
         acceptedTokenCount: Int,
-        totalTokenCount: Int
+        totalTokenCount: Int,
+        unmodifiedAcceptedTokenCount: Int?
     ): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
         requestBuilder.telemetryEvent { telemetryEventBuilder ->
             telemetryEventBuilder.codeCoverageEvent {
@@ -306,6 +308,7 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                 it.acceptedCharacterCount(acceptedTokenCount)
                 it.totalCharacterCount(totalTokenCount)
                 it.timestamp(Instant.now())
+                it.unmodifiedAcceptedCharacterCount(unmodifiedAcceptedTokenCount)
             }
         }
         requestBuilder.optOutPreference(getTelemetryOptOutPreference())

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt

@@ -85,6 +85,10 @@ abstract class CodeWhispererCodeCoverageTracker(
                     rangeMarkers.add(rangeMarker)
                     val originalRecommendation = extractRangeMarkerString(rangeMarker) ?: return
                     rangeMarker.putUserData(KEY_REMAINING_RECOMMENDATION, originalRecommendation)
+                    runReadAction {
+                        // also increment total tokens because accepted tokens are part of it
+                        incrementTotalTokens(rangeMarker.document, originalRecommendation.length)
+                    }
                 }
             }
         )
@@ -112,12 +116,16 @@ abstract class CodeWhispererCodeCoverageTracker(
             LOG.debug { "event with isWholeTextReplaced flag: $event" }
             if (event.oldTimeStamp == 0L) return
         }
-        // This case capture IDE reformatting the document, which will be blank string
-        if (isDocumentEventFromReformatting(event)) return
-
-        // Don't capture deletion events
-        if (event.newLength <= event.oldLength) return
-        incrementTotalTokens(event.document, event.newLength - event.oldLength)
+        // only count total tokens when it is a user keystroke input
+        // do not count doc changes from copy & paste of >=2 characters
+        // do not count other changes from formatter, git command, etc
+        // edge case: event can be from user hit enter with indentation where change is \n\t\t, count as 1 char increase in total chars
+        // when event is auto closing [{(', there will be 2 separated events, both count as 1 char increase in total chars
+        val text = event.newFragment.toString()
+        if ((event.newLength == 1 && event.oldLength == 0) || (text.startsWith('\n') && text.trim().isEmpty())) {
+            incrementTotalTokens(event.document, 1)
+            return
+        }
     }
 
     internal fun extractRangeMarkerString(rangeMarker: RangeMarker): String? = runReadAction {
@@ -173,9 +181,6 @@ abstract class CodeWhispererCodeCoverageTracker(
         }
     }
 
-    private fun isDocumentEventFromReformatting(event: DocumentEvent): Boolean =
-        (event.newFragment.toString().isBlank() && event.oldFragment.toString().isBlank()) && (event.oldLength == 0 || event.newLength == 0)
-
     private fun reset() {
         startTime = Instant.now()
         rangeMarkers.clear()
@@ -184,8 +189,12 @@ abstract class CodeWhispererCodeCoverageTracker(
     }
 
     internal fun emitCodeWhispererCodeContribution() {
-        // If the user is inactive, don't emit the telemetry
+        // If the user is inactive or did not invoke, don't emit the telemetry
         if (percentage == null) return
+        if (myServiceInvocationCount.get() <= 0) return
+
+        // accepted char count without considering modification
+        var rawAcceptedCharacterCount = 0
         rangeMarkers.forEach { rangeMarker ->
             if (!rangeMarker.isValid) return@forEach
             // if users add more code upon the recommendation generated from CodeWhisperer, we consider those added part as userToken but not CwsprTokens
@@ -199,6 +208,7 @@ abstract class CodeWhispererCodeCoverageTracker(
                 }
                 return@forEach
             }
+            rawAcceptedCharacterCount += originalRecommendation.length
             val delta = getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation)
             runReadAction {
                 incrementAcceptedTokens(rangeMarker.document, delta)
@@ -207,12 +217,14 @@ abstract class CodeWhispererCodeCoverageTracker(
         val customizationArn: String? = CodeWhispererModelConfigurator.getInstance().activeCustomization(project)?.arn
 
         runIfIdcConnectionOrTelemetryEnabled(project) {
+            // here acceptedTokensSize is the count of accepted chars post user modification
             try {
                 val response = CodeWhispererClientAdaptor.getInstance(project).sendCodePercentageTelemetry(
                     language,
                     customizationArn,
-                    acceptedTokensSize,
-                    totalTokensSize
+                    rawAcceptedCharacterCount,
+                    totalTokensSize,
+                    acceptedTokensSize
                 )
                 LOG.debug { "Successfully sent code percentage telemetry. RequestId: ${response.responseMetadata().requestId()}" }
             } catch (e: Exception) {

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererClientAdaptorTest.kt

@@ -387,7 +387,7 @@ class CodeWhispererClientAdaptorTest {
     @Test
     fun `sendTelemetryEvent for codePercentage respects telemetry optin status`() {
         sendTelemetryEventOptOutCheckHelper {
-            sut.sendCodePercentageTelemetry(aProgrammingLanguage(), aString(), 0, 1)
+            sut.sendCodePercentageTelemetry(aProgrammingLanguage(), aString(), 0, 1, 0)
         }
     }
 

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt

@@ -37,6 +37,7 @@ import software.aws.toolkits.core.telemetry.TelemetryBatcher
 import software.aws.toolkits.core.telemetry.TelemetryPublisher
 import software.aws.toolkits.core.utils.test.aString
 import software.aws.toolkits.jetbrains.core.MockClientManagerRule
+import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.keystrokeInput
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.pythonFileName
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.pythonResponse
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.pythonTestLeftContext
@@ -78,8 +79,9 @@ internal abstract class CodeWhispererCodeCoverageTrackerTestBase(myProjectRule:
         timeWindowInSec: Long,
         language: CodeWhispererProgrammingLanguage,
         rangeMarkers: MutableList<RangeMarker> = mutableListOf(),
-        codeCoverageTokens: MutableMap<Document, CodeCoverageTokens> = mutableMapOf()
-    ) : CodeWhispererCodeCoverageTracker(project, timeWindowInSec, language, rangeMarkers, codeCoverageTokens, AtomicInteger(0))
+        codeCoverageTokens: MutableMap<Document, CodeCoverageTokens> = mutableMapOf(),
+        invocationCount: Int = 0,
+    ) : CodeWhispererCodeCoverageTracker(project, timeWindowInSec, language, rangeMarkers, codeCoverageTokens, AtomicInteger(invocationCount))
 
     protected class TestTelemetryService(
         publisher: TelemetryPublisher = NoOpPublisher(),
@@ -230,6 +232,24 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
         CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut
         sut.activateTrackerIfNotActive()
 
+        fixture.configureByText(pythonFileName, "")
+        runInEdtAndWait {
+            WriteCommandAction.runWriteCommandAction(project) {
+                fixture.editor.appendString(keystrokeInput)
+            }
+        }
+        val captor = argumentCaptor<DocumentEvent>()
+        verify(sut, Times(1)).documentChanged(captor.capture())
+        assertThat(captor.firstValue.newFragment.toString()).isEqualTo(keystrokeInput)
+        assertThat(sut.totalTokensSize).isEqualTo(keystrokeInput.length)
+    }
+
+    @Test
+    fun `test tracker is not listening to multi char input and will not increment totalTokens - add new code`() {
+        sut = spy(TestCodePercentageTracker(project, TOTAL_SECONDS_IN_MINUTE, CodeWhispererPython.INSTANCE))
+        CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut
+        sut.activateTrackerIfNotActive()
+
         fixture.configureByText(pythonFileName, "")
         runInEdtAndWait {
             WriteCommandAction.runWriteCommandAction(project) {
@@ -239,16 +259,15 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
         val captor = argumentCaptor<DocumentEvent>()
         verify(sut, Times(1)).documentChanged(captor.capture())
         assertThat(captor.firstValue.newFragment.toString()).isEqualTo(pythonTestLeftContext)
-        val oldSize = sut.totalTokensSize
-        assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length)
+        assertThat(sut.totalTokensSize).isEqualTo(0)
 
         val anotherCode = "(x, y):"
         runInEdtAndWait {
             WriteCommandAction.runWriteCommandAction(project) {
                 fixture.editor.appendString(anotherCode)
             }
         }
-        assertThat(sut.totalTokensSize).isEqualTo(oldSize + anotherCode.length)
+        assertThat(sut.totalTokensSize).isEqualTo(0)
     }
 
     @Test
@@ -275,7 +294,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
     }
 
     @Test
-    fun `test tracker documentChanged - will not increment tokens on blank string of length greater than 1`() {
+    fun `test tracker documentChanged - will increment tokens on user input blank string`() {
         sut = TestCodePercentageTracker(
             project,
             TOTAL_SECONDS_IN_MINUTE,
@@ -292,7 +311,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
             }
         }
 
-        assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length)
+        assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length + 1)
     }
 
     @Test
@@ -394,20 +413,19 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
             on { getUserData(any<Key<String>>()) } doReturn "foo"
             on { document } doReturn fixture.editor.document
         }
-
         sut = spy(
             TestCodePercentageTracker(
                 project,
                 TOTAL_SECONDS_IN_MINUTE,
                 CodeWhispererPython.INSTANCE,
                 mutableListOf(rangeMarkerMock1),
-                mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalTokens = 100, acceptedTokens = 0))
+                mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalTokens = 100, acceptedTokens = 0)),
+                1
             )
         ) {
             onGeneric { extractRangeMarkerString(any()) } doReturn "fou"
-            onGeneric { getAcceptedTokensDelta(any(), any()) } doReturn 99
+            onGeneric { getAcceptedTokensDelta(any(), any()) } doReturn 1
         }
-
         sut.emitCodeWhispererCodeContribution()
 
         val metricCaptor = argumentCaptor<MetricEvent>()
@@ -416,8 +434,8 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
             metricCaptor.allValues,
             CODE_PERCENTAGE,
             1,
-            CWSPR_PERCENTAGE to "99",
-            CWSPR_ACCEPTED_TOKENS to "99",
+            CWSPR_PERCENTAGE to "1",
+            CWSPR_ACCEPTED_TOKENS to "1",
             CWSPR_TOTAL_TOKENS to "100",
             "codewhispererUserGroup" to userGroup.name,
         )
@@ -508,7 +526,8 @@ internal class CodeWhispererCodeCoverageTrackerTestJava : CodeWhispererCodeCover
             class Answer {
                 private int knapsack(int[] w, int[] v, int c) {
                 int[][] dp = new int[w.length + 1][c + 1];
-                    for (int i = 0; i < w.length; i++) {for (int j = 0; j <= c; j++) {
+                    for (int i = 0; i < w.length; i++) {
+                    for (int j = 0; j <= c; j++) {
                                    if (j < w[i]) {
                                 dp[i + 1][j] = dp[i][j];
                         } 

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererTelemetryTest.kt

@@ -5,7 +5,6 @@ package software.aws.toolkits.jetbrains.services.codewhisperer
 
 import com.google.gson.Gson
 import com.intellij.openapi.application.ApplicationManager
-import com.intellij.openapi.application.runInEdt
 import com.intellij.openapi.command.WriteCommandAction
 import com.intellij.openapi.editor.Editor
 import com.intellij.openapi.ui.popup.JBPopup
@@ -41,6 +40,7 @@ import software.aws.toolkits.core.telemetry.MetricEvent
 import software.aws.toolkits.core.telemetry.TelemetryBatcher
 import software.aws.toolkits.core.telemetry.TelemetryPublisher
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.emptyListResponse
+import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.keystrokeInput
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.listOfEmptyRecommendationResponse
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.listOfMixedEmptyAndNonEmptyRecommendationResponse
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.pythonResponse
@@ -508,11 +508,11 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         val fixture = projectRule.fixture
         val emptyFile = fixture.addFileToProject("/anotherFile.py", "")
         // simulate users typing behavior of the following
-        // def addTwoNumbers(
+        // user hit one key stroke
         runInEdtAndWait {
             fixture.openFileInEditor(emptyFile.virtualFile)
             WriteCommandAction.runWriteCommandAction(project) {
-                fixture.editor.appendString(pythonTestLeftContext)
+                fixture.editor.appendString(keystrokeInput)
             }
         }
         // simulate users accepting the recommendation and delete part of the recommendation
@@ -533,7 +533,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         CodeWhispererCodeCoverageTracker.getInstance(project, CodeWhispererPython.INSTANCE).dispose()
 
         val acceptedTokensSize = pythonResponse.completions()[0].content().length - deletedTokenByUser
-        val totalTokensSize = pythonTestLeftContext.length + pythonResponse.completions()[0].content().length
+        val totalTokensSize = keystrokeInput.length + pythonResponse.completions()[0].content().length
 
         val metricCaptor = argumentCaptor<MetricEvent>()
         verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
@@ -554,24 +554,23 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         val project = projectRule.project
         val fixture = projectRule.fixture
         val emptyFile = fixture.addFileToProject("/anotherFile.py", "")
-        // simulate users typing behavior of the following
-        // def addTwoNumbers
+        // simulate users typing behavior
         runInEdtAndWait {
             fixture.openFileInEditor(emptyFile.virtualFile)
             WriteCommandAction.runWriteCommandAction(project) {
-                fixture.editor.appendString(pythonTestLeftContext)
+                fixture.editor.appendString(keystrokeInput)
             }
         }
         // simulate users accepting the recommendation
         // (x, y):\n    return x + y
-        val anotherCodeSnippet = "\ndef functionWritenByMyself():\n\tpass()"
+        val anotherKeyStrokeInput = "\n  "
         withCodeWhispererServiceInvokedAndWait {
             popupManagerSpy.popupComponents.acceptButton.doClick()
         }
 
         runInEdtAndWait {
             WriteCommandAction.runWriteCommandAction(project) {
-                fixture.editor.appendString(anotherCodeSnippet)
+                fixture.editor.appendString(anotherKeyStrokeInput)
                 val currentOffset = fixture.editor.caretModel.offset
                 // delete 1 char
                 fixture.editor.document.deleteString(currentOffset - 1, currentOffset)
@@ -581,7 +580,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         }
 
         val acceptedTokensSize = pythonResponse.completions()[0].content().length
-        val totalTokensSize = pythonTestLeftContext.length + pythonResponse.completions()[0].content().length + anotherCodeSnippet.length
+        val totalTokensSize = keystrokeInput.length + pythonResponse.completions()[0].content().length + 1
 
         val metricCaptor = argumentCaptor<MetricEvent>()
         verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
@@ -605,17 +604,21 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         val userGroup = CodeWhispererUserGroupSettings.getInstance().getUserGroup()
         val project = projectRule.project
         val fixture = projectRule.fixture
-        fixture.configureByText("/file1.py", pythonTestLeftContext)
-        runInEdt {
-            fixture.editor.caretModel.moveToOffset(fixture.editor.document.textLength)
+        val emptyFile = fixture.addFileToProject("/anotherFile.py", pythonTestLeftContext)
+        // simulate users typing behavior
+        runInEdtAndWait {
+            fixture.openFileInEditor(emptyFile.virtualFile)
+            WriteCommandAction.runWriteCommandAction(project) {
+                fixture.editor.appendString(keystrokeInput)
+            }
         }
-        val file2 = fixture.addFileToProject("./file2.py", "Pre-existing string")
 
         // accept recommendation in file1.py
         withCodeWhispererServiceInvokedAndWait {
             popupManagerSpy.popupComponents.acceptButton.doClick()
         }
 
+        val file2 = fixture.addFileToProject("./file2.py", "Pre-existing string")
         // switch to file2.py and delete code there
         runInEdtAndWait {
             fixture.openFileInEditor(file2.virtualFile)
@@ -634,8 +637,8 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
             codePercentage,
             1,
             "codewhispererAcceptedTokens" to pythonResponse.completions()[0].content().length.toString(),
-            "codewhispererTotalTokens" to pythonResponse.completions()[0].content().length.toString(),
-            "codewhispererPercentage" to "100",
+            "codewhispererTotalTokens" to (1 + pythonResponse.completions()[0].content().length).toString(),
+            "codewhispererPercentage" to "96",
             "codewhispererUserGroup" to userGroup.name,
         )
     }
@@ -663,7 +666,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         runInEdtAndWait {
             fixture.openFileInEditor(emptyFile.virtualFile)
             WriteCommandAction.runWriteCommandAction(project) {
-                fixture.editor.appendString("$pythonTestLeftContext(")
+                fixture.editor.appendString("(")
                 // add closing paren but not move the caret position, simulating IDE's behavior
                 fixture.editor.document.insertString(fixture.editor.caretModel.offset, ")")
             }
@@ -675,7 +678,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         CodeWhispererCodeCoverageTracker.getInstance(project, CodeWhispererPython.INSTANCE).dispose()
 
         val acceptedTokensSize = "x, y):\n    return x + y".length
-        val totalTokensSize = "$pythonTestLeftContext(".length + acceptedTokensSize
+        val totalTokensSize = "()".length + acceptedTokensSize
         val metricCaptor = argumentCaptor<MetricEvent>()
         verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
         assertEventsContainsFieldsAndCount(