fix(amazonq): interaction event project context telemetry boolean accuracy (#4883)

* fix interaction event project context telemetry bug

* fix unit test

Author: leigaol

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt

@@ -90,6 +90,10 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
                     ChatMessage(tabId = tabId, triggerId = triggerId, messageId = requestId, messageType = ChatMessageType.Answer, followUps = followUps)
 
                 telemetryHelper.setResponseStreamTotalTime(tabId)
+                telemetryHelper.setResponseHasProjectContext(
+                    requestId,
+                    telemetryHelper.getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty()
+                )
                 telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode, countTotalNumberOfCodeBlocks(responseText))
                 emit(response)
             }

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt

@@ -43,6 +43,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
     private val responseStreamStartTime: MutableMap<String, Instant> = mutableMapOf()
     private val responseStreamTotalTime: MutableMap<String, Int> = mutableMapOf()
     private val responseStreamTimeForChunks: MutableMap<String, MutableList<Instant>> = mutableMapOf()
+    private val responseHasProjectContext: MutableMap<String, Boolean> = mutableMapOf()
 
     private val customization: CodeWhispererCustomization?
         get() = CodeWhispererModelConfigurator.getInstance().activeCustomization(context.project)
@@ -65,7 +66,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
         TriggerType.ContextMenu, TriggerType.Hotkeys -> CwsprChatTriggerInteraction.ContextMenu
     }
 
-    private fun getIsProjectContextEnabled() = CodeWhispererSettings.getInstance().isProjectContextEnabled()
+    fun getIsProjectContextEnabled() = CodeWhispererSettings.getInstance().isProjectContextEnabled()
 
     // When chat panel is focused
     fun recordEnterFocusChat() {
@@ -116,7 +117,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
             cwsprChatConversationType = CwsprChatConversationType.Chat,
             credentialStartUrl = getStartUrl(context.project),
             codewhispererCustomizationArn = data.customization?.arn,
-            cwsprChatHasProjectContext = getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty()
+            cwsprChatHasProjectContext = getMessageHasProjectContext(response.messageId)
         )
 
         val programmingLanguage = data.activeFileContext.fileContext?.fileLanguage
@@ -135,7 +136,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
             data.message.length,
             responseLength,
             numberOfCodeBlocks,
-            getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty(),
+            getMessageHasProjectContext(response.messageId),
             data.customization
         ).also {
             logger.debug {
@@ -173,7 +174,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                         else -> CwsprChatInteractionType.Unknown
                     },
                     credentialStartUrl = getStartUrl(context.project),
-                    cwsprChatHasProjectContext = getIsProjectContextEnabled()
+                    cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
@@ -185,6 +186,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                             else -> ChatMessageInteractionType.UNKNOWN_TO_SDK_VERSION
                         }
                     )
+                    hasProjectLevelContext(getMessageHasProjectContext(message.messageId))
                 }.build()
             }
 
@@ -194,12 +196,13 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     cwsprChatMessageId = message.messageId.orEmpty(),
                     cwsprChatInteractionType = CwsprChatInteractionType.ClickFollowUp,
                     credentialStartUrl = getStartUrl(context.project),
-                    cwsprChatHasProjectContext = getIsProjectContextEnabled()
+                    cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId.orEmpty())
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
                     messageId(message.messageId.orEmpty())
                     interactionType(ChatMessageInteractionType.CLICK_FOLLOW_UP)
+                    hasProjectLevelContext(getMessageHasProjectContext(message.messageId.orEmpty()))
                 }.build()
             }
 
@@ -214,14 +217,15 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     credentialStartUrl = getStartUrl(context.project),
                     cwsprChatCodeBlockIndex = message.codeBlockIndex,
                     cwsprChatTotalCodeBlocks = message.totalCodeBlocks,
-                    cwsprChatHasProjectContext = getIsProjectContextEnabled()
+                    cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
                     messageId(message.messageId)
                     interactionType(ChatMessageInteractionType.COPY_SNIPPET)
                     interactionTarget(message.insertionTargetType)
                     acceptedCharacterCount(message.code.length)
+                    hasProjectLevelContext(getMessageHasProjectContext(message.messageId))
                 }.build()
             }
 
@@ -237,7 +241,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     credentialStartUrl = getStartUrl(context.project),
                     cwsprChatCodeBlockIndex = message.codeBlockIndex,
                     cwsprChatTotalCodeBlocks = message.totalCodeBlocks,
-                    cwsprChatHasProjectContext = getIsProjectContextEnabled()
+                    cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
@@ -246,6 +250,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     interactionTarget(message.insertionTargetType)
                     acceptedCharacterCount(message.code.length)
                     acceptedLineCount(message.code.lines().size)
+                    hasProjectLevelContext(getMessageHasProjectContext(message.messageId))
                 }.build()
             }
 
@@ -265,7 +270,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     cwsprChatInteractionTarget = message.link,
                     cwsprChatHasReference = null,
                     credentialStartUrl = getStartUrl(context.project),
-                    cwsprChatHasProjectContext = getIsProjectContextEnabled()
+                    cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
@@ -278,6 +283,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                         }
                     )
                     interactionTarget(message.link)
+                    hasProjectLevelContext(getMessageHasProjectContext(message.messageId))
                 }.build()
             }
 
@@ -364,6 +370,12 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
         responseStreamTotalTime[tabId] = totalTime
     }
 
+    fun setResponseHasProjectContext(messageId: String, hasProjectContext: Boolean) {
+        responseHasProjectContext[messageId] = hasProjectContext
+    }
+
+    private fun getMessageHasProjectContext(messageId: String) = responseHasProjectContext.getOrDefault(messageId, false)
+
     @VisibleForTesting
     fun getResponseStreamTimeToFirstChunk(tabId: String): Double {
         val chunkTimes = responseStreamTimeForChunks[tabId] ?: return 0.0

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt

@@ -331,6 +331,7 @@ class TelemetryHelperTest {
                     messageId(messageId)
                     interactionType(ChatMessageInteractionType.UPVOTE)
                     customizationArn(customizationArn)
+                    hasProjectLevelContext(false)
                 }.build()
             )
         )
@@ -362,6 +363,7 @@ class TelemetryHelperTest {
         }
 
         runBlocking {
+            sut.setResponseHasProjectContext(messageId, true)
             sut.recordInteractWithMessage(IncomingCwcMessage.FollowupClicked(mock(), tabId, messageId, "command", "tabType"))
         }
 
@@ -373,6 +375,7 @@ class TelemetryHelperTest {
                     messageId(messageId)
                     interactionType(ChatMessageInteractionType.CLICK_FOLLOW_UP)
                     customizationArn(customizationArn)
+                    hasProjectLevelContext(true)
                 }.build()
             )
         )
@@ -391,7 +394,7 @@ class TelemetryHelperTest {
                 )
                 .matches({ it.metadata["credentialStartUrl"] == mockUrl }, "startUrl doesn't match")
                 .matches(
-                    { it.metadata["cwsprChatHasProjectContext"] == CodeWhispererSettings.getInstance().isProjectContextEnabled().toString() },
+                    { it.metadata["cwsprChatHasProjectContext"] == "true" },
                     "hasProjectContext doesn't match"
                 )
         }
@@ -429,6 +432,7 @@ class TelemetryHelperTest {
                     interactionTarget("insertionTargetType")
                     acceptedCharacterCount("println()".length)
                     customizationArn(customizationArn)
+                    hasProjectLevelContext(false)
                 }.build()
             )
         )
@@ -462,7 +466,6 @@ class TelemetryHelperTest {
         mockClient.stub {
             on { this.sendChatInteractWithMessageTelemetry(any<ChatInteractWithMessageEvent>()) } doReturn mockSteResponse
         }
-
         val codeBlockIndex = 1
         val totalCodeBlocks = 10
         val inserTionTargetType = "insertionTargetType"
@@ -493,6 +496,7 @@ class TelemetryHelperTest {
                     acceptedCharacterCount(code.length)
                     acceptedLineCount(code.lines().size)
                     customizationArn(customizationArn)
+                    hasProjectLevelContext(false)
                 }.build()
             )
         )
@@ -552,6 +556,7 @@ class TelemetryHelperTest {
                     interactionType(ChatMessageInteractionType.CLICK_LINK)
                     interactionTarget(link)
                     customizationArn(customizationArn)
+                    hasProjectLevelContext(false)
                 }.build()
             )
         )