telemetry(amazonq chat): Adding metrics for AB Testing. (#4330)

laileni-aws · web-flow · commit fb1f9d2ae20c · 2024-04-22T21:53:02.000-07:00
diff --git a/.changes/next-release/feature-385d3c2d-cd80-47e6-9a72-d26a7e05fafc.json b/.changes/next-release/feature-385d3c2d-cd80-47e6-9a72-d26a7e05fafc.json
@@ -0,0 +1,4 @@
+{
+  "type" : "feature",
+  "description" : "Amazon Q Chat: Added additional parameters to onCopyCodeToClipboard and onCodeInsertToCursorPosition events"
+}
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt
@@ -32,6 +32,19 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
     private var requestId: String = ""
     private var statusCode: Int = 0
 
+    companion object {
+        val CODE_BLOCK_REGEX: Regex = Regex("^```", RegexOption.MULTILINE)
+    }
+
+    private fun countTotalNumberOfCodeBlocks(message: StringBuilder): Int {
+        if (message.isEmpty()) {
+            return 0
+        }
+        val countOfCodeBlocks = CODE_BLOCK_REGEX.findAll(message)
+        val numberOfTripleBackTicksInMarkdown = countOfCodeBlocks.count()
+        return numberOfTripleBackTicksInMarkdown / 2
+    }
+
     fun handle(
         tabId: String,
         triggerId: String,
@@ -75,7 +88,7 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
                     ChatMessage(tabId = tabId, triggerId = triggerId, messageId = requestId, messageType = ChatMessageType.Answer, followUps = followUps)
 
                 telemetryHelper.setResponseStreamTotalTime(tabId)
-                telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode)
+                telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode, countTotalNumberOfCodeBlocks(responseText))
                 emit(response)
             }
             .catch { exception ->
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt
@@ -83,7 +83,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
     }
 
     // When Chat API responds to a user message (full response streamed)
-    fun recordAddMessage(data: ChatRequestData, response: ChatMessage, responseLength: Int, statusCode: Int) {
+    fun recordAddMessage(data: ChatRequestData, response: ChatMessage, responseLength: Int, statusCode: Int, numberOfCodeBlocks: Int) {
         AmazonqTelemetry.addMessage(
             cwsprChatConversationId = getConversationId(response.tabId).orEmpty(),
             cwsprChatMessageId = response.messageId,
@@ -93,7 +93,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
             cwsprChatProgrammingLanguage = data.activeFileContext.fileContext?.fileLanguage,
             cwsprChatActiveEditorTotalCharacters = data.activeFileContext.focusAreaContext?.codeSelection?.length,
             cwsprChatActiveEditorImportCount = data.activeFileContext.focusAreaContext?.codeNames?.fullyQualifiedNames?.used?.size,
-            cwsprChatResponseCodeSnippetCount = 0,
+            cwsprChatResponseCodeSnippetCount = numberOfCodeBlocks,
             cwsprChatResponseCode = statusCode,
             cwsprChatSourceLinkCount = response.relatedSuggestions?.size,
             cwsprChatReferencesCount = 0, // TODO
@@ -122,6 +122,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
             (responseStreamTotalTime[response.tabId] ?: 0).toDouble(),
             data.message.length,
             responseLength,
+            numberOfCodeBlocks
         )
     }
 
@@ -190,7 +191,9 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     cwsprChatAcceptedCharactersLength = message.code.length,
                     cwsprChatInteractionTarget = message.insertionTargetType,
                     cwsprChatHasReference = null,
-                    credentialStartUrl = getStartUrl(context.project)
+                    credentialStartUrl = getStartUrl(context.project),
+                    cwsprChatCodeBlockIndex = message.codeBlockIndex,
+                    cwsprChatTotalCodeBlocks = message.totalCodeBlocks
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
@@ -210,7 +213,9 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
                     cwsprChatAcceptedNumberOfLines = message.code.lines().size,
                     cwsprChatInteractionTarget = message.insertionTargetType,
                     cwsprChatHasReference = null,
-                    credentialStartUrl = getStartUrl(context.project)
+                    credentialStartUrl = getStartUrl(context.project),
+                    cwsprChatCodeBlockIndex = message.codeBlockIndex,
+                    cwsprChatTotalCodeBlocks = message.totalCodeBlocks
                 )
                 ChatInteractWithMessageEvent.builder().apply {
                     conversationId(getConversationId(message.tabId).orEmpty())
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/messages/CwcMessage.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/messages/CwcMessage.kt
@@ -67,6 +67,9 @@ sealed interface IncomingCwcMessage : CwcMessage {
         val messageId: String,
         val code: String,
         val insertionTargetType: String?,
+        val eventId: String?,
+        val codeBlockIndex: Int?,
+        val totalCodeBlocks: Int?
     ) : IncomingCwcMessage
 
     data class InsertCodeAtCursorPosition(
@@ -75,6 +78,9 @@ sealed interface IncomingCwcMessage : CwcMessage {
         val code: String,
         val insertionTargetType: String?,
         val codeReference: List<CodeReference>?,
+        val eventId: String?,
+        val codeBlockIndex: Int?,
+        val totalCodeBlocks: Int?
     ) : IncomingCwcMessage
 
     data class TriggerTabIdReceived(
diff --git a/plugins/amazonq/mynah-ui/package-lock.json b/plugins/amazonq/mynah-ui/package-lock.json
diff --git a/plugins/amazonq/mynah-ui/package.json b/plugins/amazonq/mynah-ui/package.json
@@ -12,7 +12,7 @@
         "lintfix": "eslint -c .eslintrc.js --fix --ext .ts ."
     },
     "dependencies": {
-        "@aws/mynah-ui-chat": "npm:@aws/mynah-ui@4.4.2",
+        "@aws/mynah-ui-chat": "npm:@aws/mynah-ui@4.5.6",
         "@types/node": "^14.18.5",
         "fs-extra": "^10.0.1",
         "ts-node": "^10.7.0",
diff --git a/plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/cwChatConnector.ts b/plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/cwChatConnector.ts
@@ -92,7 +92,10 @@ export class Connector {
         messageId: string,
         code?: string,
         type?: 'selection' | 'block',
-        codeReference?: CodeReference[]
+        codeReference?: CodeReference[],
+        eventId?: string,
+        codeBlockIndex?: number,
+        totalCodeBlocks?: number
     ): void => {
         this.sendMessageToExtension({
             tabID: tabID,
@@ -102,6 +105,9 @@ export class Connector {
             tabType: 'cwc',
             insertionTargetType: type,
             codeReference,
+            eventId,
+            codeBlockIndex,
+            totalCodeBlocks,
         })
     }
 
@@ -110,7 +116,10 @@ export class Connector {
         messageId: string,
         code?: string,
         type?: 'selection' | 'block',
-        codeReference?: CodeReference[]
+        codeReference?: CodeReference[],
+        eventId?: string,
+        codeBlockIndex?: number,
+        totalCodeBlocks?: number
     ): void => {
         this.sendMessageToExtension({
             tabID: tabID,
@@ -120,6 +129,9 @@ export class Connector {
             tabType: 'cwc',
             insertionTargetType: type,
             codeReference,
+            eventId,
+            codeBlockIndex,
+            totalCodeBlocks,
         })
     }
 
diff --git a/plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts b/plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts
@@ -212,11 +212,14 @@ export class Connector {
         messageId: string,
         code?: string,
         type?: 'selection' | 'block',
-        codeReference?: CodeReference[]
+        codeReference?: CodeReference[],
+        eventId?: string,
+        codeBlockIndex?: number,
+        totalCodeBlocks?: number
     ): void => {
         switch (this.tabsStorage.getTab(tabID)?.type) {
             case 'cwc':
-                this.cwChatConnector.onCodeInsertToCursorPosition(tabID, messageId, code, type, codeReference)
+                this.cwChatConnector.onCodeInsertToCursorPosition(tabID, messageId, code, type, codeReference, eventId, codeBlockIndex, totalCodeBlocks)
                 break
             case 'featuredev':
                 this.featureDevChatConnector.onCodeInsertToCursorPosition(tabID, code, type, codeReference)
@@ -229,11 +232,14 @@ export class Connector {
         messageId: string,
         code?: string,
         type?: 'selection' | 'block',
-        codeReference?: CodeReference[]
+        codeReference?: CodeReference[],
+        eventId?: string,
+        codeBlockIndex?: number,
+        totalCodeBlocks?: number
     ): void => {
         switch (this.tabsStorage.getTab(tabID)?.type) {
             case 'cwc':
-                this.cwChatConnector.onCopyCodeToClipboard(tabID, messageId, code, type, codeReference)
+                this.cwChatConnector.onCopyCodeToClipboard(tabID, messageId, code, type, codeReference, eventId, codeBlockIndex, totalCodeBlocks)
                 break
             case 'featuredev':
                 this.featureDevChatConnector.onCopyCodeToClipboard(tabID, code, type, codeReference)
diff --git a/plugins/amazonq/mynah-ui/src/mynah-ui/ui/main.ts b/plugins/amazonq/mynah-ui/src/mynah-ui/ui/main.ts
@@ -393,8 +393,26 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
             })
         },
         onCodeInsertToCursorPosition: connector.onCodeInsertToCursorPosition,
-        onCopyCodeToClipboard: (tabId, messageId, code, type, referenceTrackerInfo) => {
-            connector.onCopyCodeToClipboard(tabId, messageId, code, type, referenceTrackerInfo)
+        onCopyCodeToClipboard: (
+            tabId,
+            messageId,
+            code,
+            type,
+            referenceTrackerInfo,
+            eventId,
+            codeBlockIndex,
+            totalCodeBlocks
+        ) => {
+            connector.onCopyCodeToClipboard(
+                tabId,
+                messageId,
+                code,
+                type,
+                referenceTrackerInfo,
+                eventId,
+                codeBlockIndex,
+                totalCodeBlocks
+            )
             mynahUI.notify({
                 type: NotificationType.SUCCESS,
                 content: 'Selected code is copied to clipboard',
diff --git a/plugins/core/jetbrains-community/resources/telemetryOverride.json b/plugins/core/jetbrains-community/resources/telemetryOverride.json
@@ -165,6 +165,16 @@
             "type": "string",
             "description": "Identifies the entity within the message that user interacts with."
         },
+        {
+            "name": "cwsprChatCodeBlockIndex",
+            "type": "int",
+            "description": "Index of the code block inside a message in the conversation."
+        },
+        {
+            "name": "cwsprChatTotalCodeBlocks",
+            "type": "int",
+            "description": "Total number of code blocks inside a message in the conversation."
+        },
         {
             "name": "cwsprChatAcceptedCharactersLength",
             "type": "int",
@@ -469,6 +479,14 @@
                     "type": "cwsprChatInteractionTarget",
                     "required": false
                 },
+                {
+                    "type": "cwsprChatCodeBlockIndex",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatTotalCodeBlocks",
+                    "required": false
+                },
                 {
                     "type": "cwsprChatAcceptedCharactersLength",
                     "required": false
diff --git a/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json b/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json
@@ -370,7 +370,8 @@
                 "timeBetweenChunks": { "shape": "TimeBetweenChunks" },
                 "fullResponselatency": { "shape": "Double" },
                 "requestLength": { "shape": "Integer" },
-                "responseLength": { "shape": "Integer" }
+                "responseLength": { "shape": "Integer" },
+                "numberOfCodeBlocks": { "shape": "Integer" }
             }
         },
         "ChatHistory":{
diff --git a/plugins/toolkit/jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt b/plugins/toolkit/jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt
@@ -149,6 +149,7 @@ interface CodeWhispererClientAdaptor : Disposable {
         fullResponselatency: Double?,
         requestLength: Int?,
         responseLength: Int?,
+        numberOfCodeBlocks: Int?
     ): SendTelemetryEventResponse
 
     fun sendChatInteractWithMessageTelemetry(
@@ -447,7 +448,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         timeBetweenChunks: List<Double>?,
         fullResponselatency: Double?,
         requestLength: Int?,
-        responseLength: Int?
+        responseLength: Int?,
+        numberOfCodeBlocks: Int?
     ): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
         requestBuilder.telemetryEvent { telemetryEventBuilder ->
             telemetryEventBuilder.chatAddMessageEvent {
@@ -462,6 +464,7 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                 it.fullResponselatency(fullResponselatency)
                 it.requestLength(requestLength)
                 it.responseLength(responseLength)
+                it.numberOfCodeBlocks(numberOfCodeBlocks)
             }
         }
         requestBuilder.optOutPreference(getTelemetryOptOutPreference())

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +  "type" : "feature",
 +  "description" : "Amazon Q Chat: Added additional parameters to onCopyCodeToClipboard and onCodeInsertToCursorPosition events"
 +}