diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt index 0f3754f6c25..7c2f36221f8 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt @@ -90,6 +90,10 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) { ChatMessage(tabId = tabId, triggerId = triggerId, messageId = requestId, messageType = ChatMessageType.Answer, followUps = followUps) telemetryHelper.setResponseStreamTotalTime(tabId) + telemetryHelper.setResponseHasProjectContext( + requestId, + telemetryHelper.getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty() + ) telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode, countTotalNumberOfCodeBlocks(responseText)) emit(response) } diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt index 5b0989fc594..c49592bc9e1 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt @@ -43,6 +43,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se private val responseStreamStartTime: MutableMap = mutableMapOf() private val responseStreamTotalTime: MutableMap = mutableMapOf() private val responseStreamTimeForChunks: MutableMap> = mutableMapOf() + private val responseHasProjectContext: MutableMap = mutableMapOf() private val customization: CodeWhispererCustomization? get() = CodeWhispererModelConfigurator.getInstance().activeCustomization(context.project) @@ -65,7 +66,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se TriggerType.ContextMenu, TriggerType.Hotkeys -> CwsprChatTriggerInteraction.ContextMenu } - private fun getIsProjectContextEnabled() = CodeWhispererSettings.getInstance().isProjectContextEnabled() + fun getIsProjectContextEnabled() = CodeWhispererSettings.getInstance().isProjectContextEnabled() // When chat panel is focused fun recordEnterFocusChat() { @@ -116,7 +117,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se cwsprChatConversationType = CwsprChatConversationType.Chat, credentialStartUrl = getStartUrl(context.project), codewhispererCustomizationArn = data.customization?.arn, - cwsprChatHasProjectContext = getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty() + cwsprChatHasProjectContext = getMessageHasProjectContext(response.messageId) ) val programmingLanguage = data.activeFileContext.fileContext?.fileLanguage @@ -135,7 +136,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se data.message.length, responseLength, numberOfCodeBlocks, - getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty(), + getMessageHasProjectContext(response.messageId), data.customization ).also { logger.debug { @@ -173,7 +174,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se else -> CwsprChatInteractionType.Unknown }, credentialStartUrl = getStartUrl(context.project), - cwsprChatHasProjectContext = getIsProjectContextEnabled() + cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId) ) ChatInteractWithMessageEvent.builder().apply { conversationId(getConversationId(message.tabId).orEmpty()) @@ -185,6 +186,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se else -> ChatMessageInteractionType.UNKNOWN_TO_SDK_VERSION } ) + hasProjectLevelContext(getMessageHasProjectContext(message.messageId)) }.build() } @@ -194,12 +196,13 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se cwsprChatMessageId = message.messageId.orEmpty(), cwsprChatInteractionType = CwsprChatInteractionType.ClickFollowUp, credentialStartUrl = getStartUrl(context.project), - cwsprChatHasProjectContext = getIsProjectContextEnabled() + cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId.orEmpty()) ) ChatInteractWithMessageEvent.builder().apply { conversationId(getConversationId(message.tabId).orEmpty()) messageId(message.messageId.orEmpty()) interactionType(ChatMessageInteractionType.CLICK_FOLLOW_UP) + hasProjectLevelContext(getMessageHasProjectContext(message.messageId.orEmpty())) }.build() } @@ -214,7 +217,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se credentialStartUrl = getStartUrl(context.project), cwsprChatCodeBlockIndex = message.codeBlockIndex, cwsprChatTotalCodeBlocks = message.totalCodeBlocks, - cwsprChatHasProjectContext = getIsProjectContextEnabled() + cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId) ) ChatInteractWithMessageEvent.builder().apply { conversationId(getConversationId(message.tabId).orEmpty()) @@ -222,6 +225,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se interactionType(ChatMessageInteractionType.COPY_SNIPPET) interactionTarget(message.insertionTargetType) acceptedCharacterCount(message.code.length) + hasProjectLevelContext(getMessageHasProjectContext(message.messageId)) }.build() } @@ -237,7 +241,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se credentialStartUrl = getStartUrl(context.project), cwsprChatCodeBlockIndex = message.codeBlockIndex, cwsprChatTotalCodeBlocks = message.totalCodeBlocks, - cwsprChatHasProjectContext = getIsProjectContextEnabled() + cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId) ) ChatInteractWithMessageEvent.builder().apply { conversationId(getConversationId(message.tabId).orEmpty()) @@ -246,6 +250,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se interactionTarget(message.insertionTargetType) acceptedCharacterCount(message.code.length) acceptedLineCount(message.code.lines().size) + hasProjectLevelContext(getMessageHasProjectContext(message.messageId)) }.build() } @@ -265,7 +270,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se cwsprChatInteractionTarget = message.link, cwsprChatHasReference = null, credentialStartUrl = getStartUrl(context.project), - cwsprChatHasProjectContext = getIsProjectContextEnabled() + cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId) ) ChatInteractWithMessageEvent.builder().apply { conversationId(getConversationId(message.tabId).orEmpty()) @@ -278,6 +283,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se } ) interactionTarget(message.link) + hasProjectLevelContext(getMessageHasProjectContext(message.messageId)) }.build() } @@ -364,6 +370,12 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se responseStreamTotalTime[tabId] = totalTime } + fun setResponseHasProjectContext(messageId: String, hasProjectContext: Boolean) { + responseHasProjectContext[messageId] = hasProjectContext + } + + private fun getMessageHasProjectContext(messageId: String) = responseHasProjectContext.getOrDefault(messageId, false) + @VisibleForTesting fun getResponseStreamTimeToFirstChunk(tabId: String): Double { val chunkTimes = responseStreamTimeForChunks[tabId] ?: return 0.0 diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt index 01310841b56..f8878f68b47 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt @@ -331,6 +331,7 @@ class TelemetryHelperTest { messageId(messageId) interactionType(ChatMessageInteractionType.UPVOTE) customizationArn(customizationArn) + hasProjectLevelContext(false) }.build() ) ) @@ -362,6 +363,7 @@ class TelemetryHelperTest { } runBlocking { + sut.setResponseHasProjectContext(messageId, true) sut.recordInteractWithMessage(IncomingCwcMessage.FollowupClicked(mock(), tabId, messageId, "command", "tabType")) } @@ -373,6 +375,7 @@ class TelemetryHelperTest { messageId(messageId) interactionType(ChatMessageInteractionType.CLICK_FOLLOW_UP) customizationArn(customizationArn) + hasProjectLevelContext(true) }.build() ) ) @@ -391,7 +394,7 @@ class TelemetryHelperTest { ) .matches({ it.metadata["credentialStartUrl"] == mockUrl }, "startUrl doesn't match") .matches( - { it.metadata["cwsprChatHasProjectContext"] == CodeWhispererSettings.getInstance().isProjectContextEnabled().toString() }, + { it.metadata["cwsprChatHasProjectContext"] == "true" }, "hasProjectContext doesn't match" ) } @@ -429,6 +432,7 @@ class TelemetryHelperTest { interactionTarget("insertionTargetType") acceptedCharacterCount("println()".length) customizationArn(customizationArn) + hasProjectLevelContext(false) }.build() ) ) @@ -462,7 +466,6 @@ class TelemetryHelperTest { mockClient.stub { on { this.sendChatInteractWithMessageTelemetry(any()) } doReturn mockSteResponse } - val codeBlockIndex = 1 val totalCodeBlocks = 10 val inserTionTargetType = "insertionTargetType" @@ -493,6 +496,7 @@ class TelemetryHelperTest { acceptedCharacterCount(code.length) acceptedLineCount(code.lines().size) customizationArn(customizationArn) + hasProjectLevelContext(false) }.build() ) ) @@ -552,6 +556,7 @@ class TelemetryHelperTest { interactionType(ChatMessageInteractionType.CLICK_LINK) interactionTarget(link) customizationArn(customizationArn) + hasProjectLevelContext(false) }.build() ) )