Send empty telemetry event for empty recommendations (#3245)

1. There are two types of empty recommendations server could
return to the client. The first one is an empty list of recommendations.
The second one is a list of recommendations but some recommendations
are empty (strings). Previously we don't send user decision events
for both cases. This is resulting in a discrepancy between serviceInvocation
event and userDecision event per codewhispererSessionId. We decided to send
userDecision events for both. Additionally, for empty list, we just send
the event and stop early, since there isn't additonal things to render.
For empty recommendations, they will be kept in the state but not shown
to the users, and userDecision events for them will be sent later when
we send events for all recommendations we received on session close.

Author: andrewyuq

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/popup/CodeWhispererPopupManager.kt

@@ -570,6 +570,7 @@ class CodeWhispererPopupManager {
 
     private fun isValidRecommendation(detailContext: DetailContext, userInput: String, typeahead: String): Boolean {
         if (detailContext.isDiscarded) return false
+        if (detailContext.recommendation.content().isEmpty()) return false
         val indexOfFirstNonWhiteSpace = typeahead.indexOfFirst { !it.isWhitespace() }
         if (indexOfFirstNonWhiteSpace == -1) return true
 

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt

@@ -59,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhisperer
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.checkEmptyRecommendations
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.CodewhispererCompletionType
+import software.aws.toolkits.telemetry.CodewhispererSuggestionState
 import software.aws.toolkits.telemetry.CodewhispererTriggerType
 import java.util.concurrent.TimeUnit
 
@@ -151,42 +152,6 @@ class CodeWhispererService {
                         null
                     )
 
-                    if (emptyRecommendations) {
-                        // In case the server returns a response with empty recommendations, client-side should not
-                        // show them. They will not be added to the worker queue or appear in any of the user decision.
-                        // So we should continue to the next one or stop if there's no next one.
-                        LOG.debug { "Received 0 recommendation from response, requestId: $requestId" }
-                        if (response.nextToken().isNotEmpty()) {
-                            // If job is cancelled before we do another request, don't bother making
-                            // another API call to save resources
-                            if (!isActive) {
-                                LOG.debug { "Skipping sending remaining requests on CodeWhisperer session exit" }
-                                break
-                            }
-                            continue
-                        }
-
-                        // At this point, the current recommendation is empty and there's no next request,
-                        // We should choose to display the "no recommendations" hint when it's a manual trigger and
-                        // There's no active CodeWhisperer popup, and we should do these in EDT.
-                        runInEdt {
-                            CodeWhispererInvocationStatus.getInstance().finishInvocation()
-                            if (CodeWhispererInvocationStatus.getInstance().isPopupActive()) {
-                                states?.let {
-                                    CodeWhispererPopupManager.getInstance().updatePopupPanel(
-                                        it,
-                                        CodeWhispererPopupManager.getInstance().sessionContext
-                                    )
-                                }
-                                return@runInEdt
-                            }
-                            CodeWhispererPopupManager.getInstance().cancelPopup(popup)
-                            if (requestContext.triggerTypeInfo.triggerType != CodewhispererTriggerType.OnDemand) return@runInEdt
-                            showCodeWhispererInfoHint(requestContext.editor, message("codewhisperer.popup.no_recommendations"))
-                        }
-                        break
-                    }
-
                     val validatedResponse = validateResponse(response)
 
                     runInEdt {
@@ -294,13 +259,14 @@ class CodeWhispererService {
         val responseContext = workerContext.responseContext
         val response = workerContext.response
         val popup = workerContext.popup
+        val requestId = response.responseMetadata().requestId()
 
         // At this point when we are in EDT, the state of the popup will be thread-safe
         // across this thread execution, so if popup is disposed, we will stop here.
         // This extra check is needed because there's a time between when we get the response and
         // when we enter the EDT.
         if (popup.isDisposed) {
-            LOG.debug { "Stop showing CodeWhisperer recommendations on CodeWhisperer session exit" }
+            LOG.debug { "Stop showing CodeWhisperer recommendations on CodeWhisperer session exit. RequestId: $requestId" }
             return null
         }
 
@@ -312,35 +278,50 @@ class CodeWhispererService {
             requestContext.editor,
             requestContext.caretPosition
         )
-        val popupIsShowing: Boolean
+        val isPopupShowing: Boolean
         val nextStates: InvocationContext?
         if (currStates == null) {
             // first response
             nextStates = initStates(requestContext, responseContext, response, caretMovement, popup)
-            popupIsShowing = true
+            isPopupShowing = false
 
             // receiving a null state means caret has moved backward or there's a conflict with
             // Intellisense popup, so we are going to cancel the job
             if (nextStates == null) {
-                LOG.debug { "Cancelling popup and exiting CodeWhisperer session" }
+                LOG.debug { "Cancelling popup and exiting CodeWhisperer session. RequestId: $requestId" }
                 CodeWhispererPopupManager.getInstance().cancelPopup(popup)
                 return null
             }
         } else {
             // subsequent responses
             nextStates = updateStates(currStates, response)
-            popupIsShowing = checkRecommendationsValidity(currStates, false)
+            isPopupShowing = checkRecommendationsValidity(currStates, false)
         }
 
         val hasAtLeastOneValid = checkRecommendationsValidity(nextStates, response.nextToken().isEmpty())
+
+        // If there are no recommendations in this response, we need to manually send the user decision event here
+        // since it won't be sent automatically later
+        if (response.recommendations().isEmpty()) {
+            LOG.debug { "Received an empty list from response, requestId: $requestId" }
+            CodeWhispererTelemetryService.getInstance().sendUserDecisionEvent(
+                requestId,
+                requestContext,
+                responseContext,
+                Recommendation.builder().build(),
+                -1,
+                CodewhispererSuggestionState.Empty,
+                nextStates.recommendationContext.details.size
+            )
+        }
         if (!hasAtLeastOneValid) {
             if (response.nextToken().isEmpty()) {
                 LOG.debug { "None of the recommendations are valid, exiting CodeWhisperer session" }
                 CodeWhispererPopupManager.getInstance().cancelPopup(popup)
                 return null
             }
-        } else {
-            updateCodeWhisperer(nextStates, !popupIsShowing)
+        } else if (response.recommendations().isNotEmpty()) {
+            updateCodeWhisperer(nextStates, isPopupShowing)
         }
         return nextStates
     }
@@ -426,16 +407,23 @@ class CodeWhispererService {
                 !CodeWhispererSettings.getInstance().isIncludeCodeWithReference()
         }
 
-        // set to true when at least one is not discarded
-        val hasAtLeastOneValid = details.any { !it.isDiscarded }
+        // set to true when at least one is not discarded or empty
+        val hasAtLeastOneValid = details.any { !it.isDiscarded && it.recommendation.content().isNotEmpty() }
 
-        // show popup hint when non is valid and at least one is discarded by reference filter
-        if (!hasAtLeastOneValid && hasAtLeastOneDiscardedByReferenceFilter && showHint) {
-            HintManager.getInstance().showInformationHint(
-                states.requestContext.editor,
-                message("codewhisperer.popup.reference.filter"),
-                HintManager.UNDER
-            )
+        if (!hasAtLeastOneValid && showHint) {
+            if (hasAtLeastOneDiscardedByReferenceFilter) {
+                // show popup hint for filter when none is valid and at least one is discarded by reference filter
+                showCodeWhispererInfoHint(
+                    states.requestContext.editor,
+                    message("codewhisperer.popup.reference.filter")
+                )
+            } else if (states.requestContext.triggerTypeInfo.triggerType == CodewhispererTriggerType.OnDemand) {
+                // show popup hint for no recommendation when none is valid and no one is discarded by reference filter
+                showCodeWhispererInfoHint(
+                    states.requestContext.editor,
+                    message("codewhisperer.popup.no_recommendations")
+                )
+            }
         }
         return hasAtLeastOneValid
     }

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererTelemetryService.kt

@@ -75,7 +75,7 @@ class CodeWhispererTelemetryService {
         )
     }
 
-    private fun sendUserDecisionEvent(
+    fun sendUserDecisionEvent(
         requestId: String,
         requestContext: RequestContext,
         responseContext: ResponseContext,
@@ -185,7 +185,8 @@ class CodeWhispererTelemetryService {
                 sessionContext.seen.contains(index),
                 hasUserAccepted,
                 isDiscarded,
-                detail.hasReferences()
+                detail.hasReferences(),
+                detail.content().isEmpty()
             )
             sendUserDecisionEvent(requestId, requestContext, responseContext, detail, index, suggestionState, detailContexts.size)
         }
@@ -197,9 +198,12 @@ class CodeWhispererTelemetryService {
         hasSeen: Boolean,
         hasUserAccepted: Boolean,
         isDiscarded: Boolean,
-        hasReference: Boolean
+        hasReference: Boolean,
+        isEmpty: Boolean
     ): CodewhispererSuggestionState =
-        if (!CodeWhispererSettings.getInstance().isIncludeCodeWithReference() && hasReference) {
+        if (isEmpty) {
+            CodewhispererSuggestionState.Empty
+        } else if (!CodeWhispererSettings.getInstance().isIncludeCodeWithReference() && hasReference) {
             CodewhispererSuggestionState.Filter
         } else if (isDiscarded) {
             CodewhispererSuggestionState.Discard

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererTelemetryTest.kt

@@ -7,6 +7,7 @@ import com.google.gson.Gson
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.command.WriteCommandAction
 import com.intellij.openapi.editor.Editor
+import com.intellij.openapi.ui.popup.JBPopup
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.testFramework.replaceService
 import com.intellij.testFramework.runInEdtAndWait
@@ -21,6 +22,7 @@ import org.mockito.kotlin.atLeast
 import org.mockito.kotlin.atLeastOnce
 import org.mockito.kotlin.doAnswer
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.never
 import org.mockito.kotlin.spy
 import org.mockito.kotlin.stub
 import org.mockito.kotlin.timeout
@@ -33,6 +35,9 @@ import software.amazon.awssdk.services.codewhisperer.model.ListRecommendationsRe
 import software.aws.toolkits.core.telemetry.MetricEvent
 import software.aws.toolkits.core.telemetry.TelemetryBatcher
 import software.aws.toolkits.core.telemetry.TelemetryPublisher
+import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.emptyListResponse
+import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.listOfEmptyRecommendationResponse
+import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.listOfMixedEmptyAndNonEmptyRecommendationResponse
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.pythonResponse
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.pythonTestLeftContext
 import software.aws.toolkits.jetbrains.services.codewhisperer.CodeWhispererTestUtil.testCodeWhispererException
@@ -520,6 +525,70 @@ class CodeWhispererTelemetryTest : CodeWhispererTestBase() {
         metricCaptor.allValues.forEach { println(it) }
     }
 
+    @Test
+    fun `test empty list of recommendations should sent 1 empty userDecision event and no popup shown`() {
+        mockClient.stub {
+            on {
+                mockClient.listRecommendations(any<ListRecommendationsRequest>())
+            } doAnswer {
+                emptyListResponse
+            }
+        }
+        invokeCodeWhispererService()
+
+        verify(popupManagerSpy, never()).showPopup(any(), any(), any(), any(), any(), any(), any(), any())
+        runInEdtAndWait {
+            val metricCaptor = argumentCaptor<MetricEvent>()
+            verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
+            assertEventsContainsFieldsAndCount(
+                metricCaptor.allValues, userDecision, 1,
+                "codewhispererSuggestionState" to CodewhispererSuggestionState.Empty.toString()
+            )
+        }
+    }
+
+    @Test
+    fun `test empty recommendations should send empty user decision events`() {
+        testSendEmptyUserDecisionEventForEmptyRecommendations(listOfEmptyRecommendationResponse)
+    }
+
+    @Test
+    fun `test a mix of empty and non-empty recommendations should send empty user decision events accordingly`() {
+        testSendEmptyUserDecisionEventForEmptyRecommendations(listOfMixedEmptyAndNonEmptyRecommendationResponse)
+    }
+
+    private fun testSendEmptyUserDecisionEventForEmptyRecommendations(response: ListRecommendationsResponse) {
+        mockClient.stub {
+            on {
+                mockClient.listRecommendations(any<ListRecommendationsRequest>())
+            } doAnswer {
+                response
+            }
+        }
+
+        invokeCodeWhispererService()
+
+        val numOfEmptyRecommendations = response.recommendations().filter { it.content().isEmpty() }.size
+        if (numOfEmptyRecommendations == response.recommendations().size) {
+            verify(popupManagerSpy, never()).showPopup(any(), any(), any(), any(), any(), any(), any(), any())
+        } else {
+            val popupCaptor = argumentCaptor<JBPopup>()
+            verify(popupManagerSpy, timeout(5000))
+                .showPopup(any(), any(), any(), any(), any(), popupCaptor.capture(), any(), any())
+            runInEdtAndWait {
+                popupManagerSpy.closePopup(popupCaptor.lastValue)
+            }
+        }
+        runInEdtAndWait {
+            val metricCaptor = argumentCaptor<MetricEvent>()
+            verify(batcher, atLeastOnce()).enqueue(metricCaptor.capture())
+            assertEventsContainsFieldsAndCount(
+                metricCaptor.allValues, userDecision, numOfEmptyRecommendations,
+                "codewhispererSuggestionState" to CodewhispererSuggestionState.Empty.toString()
+            )
+        }
+    }
+
     private fun Editor.appendString(string: String) {
         val currentOffset = caretModel.primaryCaret.offset
         document.insertString(currentOffset, string)

jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererTestUtil.kt

@@ -65,6 +65,34 @@ object CodeWhispererTestUtil {
         .responseMetadata(metadata)
         .sdkHttpResponse(sdkHttpResponse)
         .build() as ListRecommendationsResponse
+    val emptyListResponse: ListRecommendationsResponse = ListRecommendationsResponse.builder()
+        .recommendations(listOf())
+        .nextToken("")
+        .responseMetadata(metadata)
+        .sdkHttpResponse(sdkHttpResponse)
+        .build() as ListRecommendationsResponse
+    val listOfEmptyRecommendationResponse: ListRecommendationsResponse = ListRecommendationsResponse.builder()
+        .recommendations(
+            generateMockRecommendationDetail(""),
+            generateMockRecommendationDetail(""),
+            generateMockRecommendationDetail(""),
+        )
+        .nextToken("")
+        .responseMetadata(metadata)
+        .sdkHttpResponse(sdkHttpResponse)
+        .build() as ListRecommendationsResponse
+    val listOfMixedEmptyAndNonEmptyRecommendationResponse: ListRecommendationsResponse = ListRecommendationsResponse.builder()
+        .recommendations(
+            generateMockRecommendationDetail(""),
+            generateMockRecommendationDetail("test recommendation 3"),
+            generateMockRecommendationDetail(""),
+            generateMockRecommendationDetail("test recommendation 4"),
+            generateMockRecommendationDetail("test recommendation 5")
+        )
+        .nextToken("")
+        .responseMetadata(metadata)
+        .sdkHttpResponse(sdkHttpResponse)
+        .build() as ListRecommendationsResponse
 
     const val pythonFileName = "test.py"
     const val javaFileName = "test.java"