telemetry(amazonq chat): numberOfCodeBlocks

Author: laileni-aws

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt

@@ -32,6 +32,15 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
     private var requestId: String = ""
     private var statusCode: Int = 0
 
+    private fun countTotalNumberOfCodeBlocks(message: StringBuilder): Int {
+        if (message.isEmpty()) {
+            return 0
+        }
+        val countOfCodeBlocks = Regex("^```", RegexOption.MULTILINE).findAll(message)
+        val numberOfTripleBackTicksInMarkdown = countOfCodeBlocks.count()
+        return numberOfTripleBackTicksInMarkdown / 2
+    }
+
     fun handle(
         tabId: String,
         triggerId: String,
@@ -75,7 +84,7 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
                     ChatMessage(tabId = tabId, triggerId = triggerId, messageId = requestId, messageType = ChatMessageType.Answer, followUps = followUps)
 
                 telemetryHelper.setResponseStreamTotalTime(tabId)
-                telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode)
+                telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode, countTotalNumberOfCodeBlocks(responseText))
                 emit(response)
             }
             .catch { exception ->

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt

@@ -83,7 +83,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
     }
 
     // When Chat API responds to a user message (full response streamed)
-    fun recordAddMessage(data: ChatRequestData, response: ChatMessage, responseLength: Int, statusCode: Int) {
+    fun recordAddMessage(data: ChatRequestData, response: ChatMessage, responseLength: Int, statusCode: Int, numberOfCodeBlocks: Int) {
         AmazonqTelemetry.addMessage(
             cwsprChatConversationId = getConversationId(response.tabId).orEmpty(),
             cwsprChatMessageId = response.messageId,
@@ -96,7 +96,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
             cwsprChatResponseCodeSnippetCount = 0,
             cwsprChatResponseCode = statusCode,
             cwsprChatSourceLinkCount = response.relatedSuggestions?.size,
-            cwsprChatReferencesCount = 0, // TODO
+            cwsprChatReferencesCount = numberOfCodeBlocks,
             cwsprChatFollowUpCount = response.followUps?.size,
             cwsprChatTimeToFirstChunk = getResponseStreamTimeToFirstChunk(response.tabId).toInt(),
             cwsprChatTimeBetweenChunks = "[${getResponseStreamTimeBetweenChunks(response.tabId).joinToString(",")}]",
@@ -122,6 +122,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
             (responseStreamTotalTime[response.tabId] ?: 0).toDouble(),
             data.message.length,
             responseLength,
+            numberOfCodeBlocks
         )
     }
 

plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json

@@ -370,7 +370,8 @@
                 "timeBetweenChunks": { "shape": "TimeBetweenChunks" },
                 "fullResponselatency": { "shape": "Double" },
                 "requestLength": { "shape": "Integer" },
-                "responseLength": { "shape": "Integer" }
+                "responseLength": { "shape": "Integer" },
+                "numberOfCodeBlocks": { "shape": "Integer" }
             }
         },
         "ChatHistory":{

plugins/toolkit/jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt

@@ -149,6 +149,7 @@ interface CodeWhispererClientAdaptor : Disposable {
         fullResponselatency: Double?,
         requestLength: Int?,
         responseLength: Int?,
+        numberOfCodeBlocks: Int?
     ): SendTelemetryEventResponse
 
     fun sendChatInteractWithMessageTelemetry(
@@ -447,7 +448,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         timeBetweenChunks: List<Double>?,
         fullResponselatency: Double?,
         requestLength: Int?,
-        responseLength: Int?
+        responseLength: Int?,
+        numberOfCodeBlocks: Int?
     ): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
         requestBuilder.telemetryEvent { telemetryEventBuilder ->
             telemetryEventBuilder.chatAddMessageEvent {
@@ -462,6 +464,7 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                 it.fullResponselatency(fullResponselatency)
                 it.requestLength(requestLength)
                 it.responseLength(responseLength)
+                it.numberOfCodeBlocks(numberOfCodeBlocks)
             }
         }
         requestBuilder.optOutPreference(getTelemetryOptOutPreference())