[codewhisperer]: emit telemetry for perceived latency (#3298)

Author: zixlin7

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/popup/CodeWhispererPopupManager.kt

@@ -60,6 +60,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.popup.listeners.Co
 import software.aws.toolkits.jetbrains.services.codewhisperer.popup.listeners.CodeWhispererPrevButtonActionListener
 import software.aws.toolkits.jetbrains.services.codewhisperer.popup.listeners.CodeWhispererScrollListener
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererInvocationStatus
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
 import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererColorUtil.POPUP_DIM_HEX
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.POPUP_INFO_TEXT_SIZE
@@ -198,10 +199,7 @@ class CodeWhispererPopupManager {
         val caretPoint = states.requestContext.editor.offsetToXY(states.requestContext.caretPosition.offset)
         sessionContext.seen.add(sessionContext.selectedIndex)
         showPopup(
-            states.requestContext.editor,
-            states.recommendationContext.details,
-            states.recommendationContext.userInputOriginal,
-            states.recommendationContext.userInputSinceInvocation,
+            states,
             sessionContext,
             states.popup,
             caretPoint,
@@ -231,15 +229,16 @@ class CodeWhispererPopupManager {
     }
 
     fun showPopup(
-        editor: Editor,
-        detailContexts: List<DetailContext>,
-        userInputOriginal: String,
-        userInput: String,
+        states: InvocationContext,
         sessionContext: SessionContext,
         popup: JBPopup,
         p: Point,
         overlappingLinesCount: Int
     ) {
+        val editor = states.requestContext.editor
+        val detailContexts = states.recommendationContext.details
+        val userInputOriginal = states.recommendationContext.userInputOriginal
+        val userInput = states.recommendationContext.userInputSinceInvocation
         val selectedIndex = sessionContext.selectedIndex
         val typeaheadOriginal = sessionContext.typeaheadOriginal
         val typeahead = sessionContext.typeahead
@@ -294,6 +293,13 @@ class CodeWhispererPopupManager {
                 CodeInsightSettings.getInstance().AUTO_POPUP_COMPLETION_LOOKUP = originalAutoPopupCompletionLookup
             }
             popup.show(relativePopupLocationToEditor)
+            val perceivedLatency = CodeWhispererInvocationStatus.getInstance().getTimeSinceDocumentChanged()
+            CodeWhispererTelemetryService.getInstance().sendPerceivedLatencyEvent(
+                detailContexts[selectedIndex].requestId,
+                states.requestContext,
+                states.responseContext,
+                perceivedLatency
+            )
         }
         if (shouldHidePopup) {
             WindowManager.getInstance().setAlphaModeRatio(popup.popupWindow, 1f)

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererInvocationStatus.kt

@@ -66,6 +66,12 @@ class CodeWhispererInvocationStatus {
         timeAtLastDocumentChanged = Instant.now()
     }
 
+    fun getTimeSinceDocumentChanged(): Double {
+        val timeSinceDocumentChanged = Duration.between(timeAtLastInvocationComplete, Instant.now())
+        val timeInDouble = timeSinceDocumentChanged.toMillis().toDouble()
+        return timeInDouble
+    }
+
     fun hasEnoughDelayToShowCodeWhisperer(): Boolean {
         val timeCanShowCodeWhisperer = timeAtLastDocumentChanged?.plusMillis(CodeWhispererConstants.POPUP_DELAY) ?: return true
         return timeCanShowCodeWhisperer.isBefore(Instant.now())

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererTelemetryService.kt

@@ -194,6 +194,26 @@ class CodeWhispererTelemetryService {
         }
     }
 
+    fun sendPerceivedLatencyEvent(
+        requestId: String,
+        requestContext: RequestContext,
+        responseContext: ResponseContext,
+        latency: Double,
+    ) {
+        val (project, _, triggerTypeInfo) = requestContext
+        val codewhispererLanguage = requestContext.fileContextInfo.programmingLanguage.toCodeWhispererLanguage()
+        CodewhispererTelemetry.perceivedLatency(
+            project = project,
+            codewhispererCompletionType = responseContext.completionType,
+            codewhispererLanguage = codewhispererLanguage,
+            codewhispererRequestId = requestId,
+            codewhispererSessionId = responseContext.sessionId,
+            codewhispererTriggerType = triggerTypeInfo.triggerType,
+            duration = latency,
+            passive = true
+        )
+    }
+
     private fun recordSuggestionState(
         index: Int,
         selectedIndex: Int,

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererTelemetryTest.kt

@@ -687,7 +687,7 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         }
         invokeCodeWhispererService()
 
-        verify(popupManagerSpy, never()).showPopup(any(), any(), any(), any(), any(), any(), any(), any())
+        verify(popupManagerSpy, never()).showPopup(any(), any(), any(), any(), any())
         runInEdtAndWait {
             val metricCaptor = argumentCaptor<MetricEvent>()
             verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
@@ -752,11 +752,11 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
 
         val numOfEmptyRecommendations = response.recommendations().filter { it.content().isEmpty() }.size
         if (numOfEmptyRecommendations == response.recommendations().size) {
-            verify(popupManagerSpy, never()).showPopup(any(), any(), any(), any(), any(), any(), any(), any())
+            verify(popupManagerSpy, never()).showPopup(any(), any(), any(), any(), any())
         } else {
             val popupCaptor = argumentCaptor<JBPopup>()
             verify(popupManagerSpy, timeout(5000))
-                .showPopup(any(), any(), any(), any(), any(), popupCaptor.capture(), any(), any())
+                .showPopup(any(), any(), popupCaptor.capture(), any(), any())
             runInEdtAndWait {
                 popupManagerSpy.closePopup(popupCaptor.lastValue)
             }

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererTestBase.kt

@@ -103,7 +103,7 @@ open class CodeWhispererTestBase {
 
         popupManagerSpy = spy(CodeWhispererPopupManager.getInstance())
         popupManagerSpy.reset()
-        doNothing().`when`(popupManagerSpy).showPopup(any(), any(), any(), any(), any(), any(), any(), any())
+        doNothing().`when`(popupManagerSpy).showPopup(any(), any(), any(), any(), any())
         ApplicationManager.getApplication().replaceService(CodeWhispererPopupManager::class.java, popupManagerSpy, disposableRule.disposable)
 
         invocationStatusSpy = spy(CodeWhispererInvocationStatus.getInstance())
@@ -152,7 +152,7 @@ open class CodeWhispererTestBase {
         val popupCaptor = argumentCaptor<JBPopup>()
         invokeCodeWhispererService()
         verify(popupManagerSpy, timeout(5000).atLeastOnce())
-            .showPopup(any(), any(), any(), any(), any(), popupCaptor.capture(), any(), any())
+            .showPopup(any(), any(), popupCaptor.capture(), any(), any())
         val popup = popupCaptor.lastValue
 
         try {

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUserActionsTest.kt

@@ -116,7 +116,7 @@ class CodeWhispererUserActionsTest : CodeWhispererTestBase() {
         assertThat(projectRule.fixture.editor.document.text).isEqualTo(expectedFileContext)
         val popupCaptor = argumentCaptor<JBPopup>()
         verify(popupManagerSpy, timeout(5000))
-            .showPopup(any(), any(), any(), any(), any(), popupCaptor.capture(), any(), any())
+            .showPopup(any(), any(), popupCaptor.capture(), any(), any())
         runInEdtAndWait {
             popupManagerSpy.closePopup(popupCaptor.lastValue)
         }
@@ -128,7 +128,7 @@ class CodeWhispererUserActionsTest : CodeWhispererTestBase() {
         projectRule.fixture.type('\n')
         val popupCaptor = argumentCaptor<JBPopup>()
         verify(popupManagerSpy, timeout(5000).atLeast(times))
-            .showPopup(any(), any(), any(), any(), any(), popupCaptor.capture(), any(), any())
+            .showPopup(any(), any(), popupCaptor.capture(), any(), any())
         runInEdtAndWait {
             popupManagerSpy.closePopup(popupCaptor.lastValue)
         }