telemetry on thumb up/down and feedback

Author: Randall-Jiang

gradle/libs.versions.toml

@@ -27,7 +27,7 @@ mockitoKotlin = "5.4.0"
 mockk = "1.13.10"
 nimbus-jose-jwt = "9.40"
 node-gradle = "7.0.2"
-telemetryGenerator = "1.0.295"
+telemetryGenerator = "1.0.297"
 testLogger = "4.0.0"
 testRetry = "1.5.10"
 # test-only; platform provides slf4j transitively at runtime

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeTestChatApp.kt

@@ -39,6 +39,9 @@ class CodeTestChatApp(private val scope: CoroutineScope) : AmazonQApp {
             "start-test-gen" to IncomingCodeTestMessage.StartTestGen::class,
             "response-body-link-click" to IncomingCodeTestMessage.ClickedLink::class,
             "button-click" to IncomingCodeTestMessage.ButtonClicked::class,
+            "chat-item-voted" to IncomingCodeTestMessage.ChatItemVoted::class,
+            "chat-item-feedback" to IncomingCodeTestMessage.ChatItemFeedback::class,
+            "button-click" to IncomingCodeTestMessage.ButtonClicked::class,
             "auth-follow-up-was-clicked" to IncomingCodeTestMessage.AuthFollowUpWasClicked::class
         )
 
@@ -80,6 +83,8 @@ class CodeTestChatApp(private val scope: CoroutineScope) : AmazonQApp {
             is IncomingCodeTestMessage.StartTestGen -> inboundAppMessagesHandler.processStartTestGen(message)
             is IncomingCodeTestMessage.ClickedLink -> inboundAppMessagesHandler.processLinkClick(message)
             is IncomingCodeTestMessage.ButtonClicked -> inboundAppMessagesHandler.processButtonClickedMessage(message)
+            is IncomingCodeTestMessage.ChatItemVoted -> inboundAppMessagesHandler.processChatItemVoted(message)
+            is IncomingCodeTestMessage.ChatItemFeedback -> inboundAppMessagesHandler.processChatItemFeedBack(message)
             is IncomingCodeTestMessage.AuthFollowUpWasClicked -> inboundAppMessagesHandler.processAuthFollowUpClick(message)
         }
     }

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/InboundAppMessagesHandler.kt

@@ -22,5 +22,9 @@ interface InboundAppMessagesHandler {
 
     suspend fun processButtonClickedMessage(message: IncomingCodeTestMessage.ButtonClicked)
 
+    suspend fun processChatItemVoted(message: IncomingCodeTestMessage.ChatItemVoted)
+
+    suspend fun processChatItemFeedBack(message: IncomingCodeTestMessage.ChatItemFeedback)
+
     suspend fun processAuthFollowUpClick(message: IncomingCodeTestMessage.AuthFollowUpWasClicked)
 }

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/controller/CodeTestChatController.kt

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 package software.aws.toolkits.jetbrains.services.amazonqCodeTest.controller
-
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.intellij.diff.DiffContentFactory
 import com.intellij.diff.DiffManager
 import com.intellij.diff.DiffManagerEx
@@ -44,8 +44,11 @@ import software.amazon.awssdk.services.codewhispererstreaming.model.TextDocument
 import software.amazon.awssdk.services.codewhispererstreaming.model.UserInputMessage
 import software.amazon.awssdk.services.codewhispererstreaming.model.UserInputMessageContext
 import software.amazon.awssdk.services.codewhispererstreaming.model.UserIntent
+import software.amazon.awssdk.services.toolkittelemetry.model.Sentiment
 import software.aws.toolkits.core.utils.debug
 import software.aws.toolkits.core.utils.getLogger
+import software.aws.toolkits.core.utils.info
+import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.AwsClientManager
 import software.aws.toolkits.jetbrains.core.coroutines.EDT
 import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnectionManager
@@ -81,14 +84,19 @@ import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.TriggerTy
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.v1.ChatSessionV1.Companion.validLanguages
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.StaticPrompt
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.StaticTextResponse
+import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.FeedbackComment
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.ActiveFileContext
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.ActiveFileContextExtractor
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.ExtractionTriggerType
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.file.FileContext
 import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
+import software.aws.toolkits.jetbrains.services.telemetry.TelemetryService
+import software.aws.toolkits.jetbrains.utils.notifyError
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.AmazonqTelemetry
+import software.aws.toolkits.telemetry.FeatureId
+import software.aws.toolkits.telemetry.InteractionType
 import software.aws.toolkits.telemetry.MetricResult
 import software.aws.toolkits.telemetry.UiTelemetry
 import java.io.File
@@ -403,6 +411,57 @@ class CodeTestChatController(
             .build()
     }
 
+    override suspend fun processChatItemFeedBack(message: IncomingCodeTestMessage.ChatItemFeedback) {
+        LOG.debug { "$FEATURE_NAME: Processing ChatItemFeedBackMessage: ${message.comment}" }
+
+        val session = codeTestChatHelper.getActiveSession()
+
+        val comment = FeedbackComment(
+            conversationId = session.startTestGenerationRequestId,
+            userComment = message.comment.orEmpty(),
+            reason = message.selectedOption,
+            type = "testgen-chat-answer-feedback",
+            messageId = "",
+        )
+
+        try {
+            TelemetryService.getInstance().sendFeedback(
+                sentiment = Sentiment.NEGATIVE,
+                comment = objectMapper.writeValueAsString(comment),
+            )
+            LOG.info { "$FEATURE_NAME answer feedback sent: \"Negative\"" }
+        } catch (e: Throwable) {
+            e.notifyError(message("feedback.submit_failed", e))
+            LOG.warn(e) { "Failed to submit feedback" }
+            return
+        }
+    }
+
+    override suspend fun processChatItemVoted(message: IncomingCodeTestMessage.ChatItemVoted) {
+        LOG.debug { "$FEATURE_NAME: Processing ChatItemVotedMessage: $message" }
+
+        val session = codeTestChatHelper.getActiveSession()
+        when (message.vote) {
+            "upvote" -> {
+                AmazonqTelemetry.feedback(
+                    featureId = FeatureId.AmazonQTest,
+                    interactionType = InteractionType.Upvote,
+                    credentialStartUrl = getStartUrl(project = context.project),
+                    amazonqConversationId = session.startTestGenerationRequestId
+
+                )
+            }
+            "downvote" -> {
+                AmazonqTelemetry.feedback(
+                    featureId = FeatureId.AmazonQTest,
+                    interactionType = InteractionType.Downvote,
+                    credentialStartUrl = getStartUrl(project = context.project),
+                    amazonqConversationId = session.startTestGenerationRequestId
+                )
+            }
+        }
+    }
+
     override suspend fun processNewTabCreatedMessage(message: IncomingCodeTestMessage.NewTabCreated) {
         newTabOpened(message.tabId)
         LOG.debug { "$FEATURE_NAME: New tab created: $message" }
@@ -1312,5 +1371,7 @@ class CodeTestChatController(
 
     companion object {
         private val LOG = getLogger<CodeTestChatController>()
+
+        private val objectMapper = jacksonObjectMapper()
     }
 }

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/messages/CodeTestMessage.kt

@@ -95,6 +95,17 @@ sealed interface IncomingCodeTestMessage : CodeTestBaseMessage {
         @JsonProperty("actionID") val actionID: String,
     ) : IncomingCodeTestMessage
 
+    data class ChatItemVoted(
+        @JsonProperty("tabID") val tabId: String,
+        val vote: String,
+    ) : IncomingCodeTestMessage
+
+    data class ChatItemFeedback(
+        @JsonProperty("tabID") val tabId: String,
+        val selectedOption: String,
+        val comment: String?,
+    ) : IncomingCodeTestMessage
+
     data class AuthFollowUpWasClicked(
         @JsonProperty("tabID") val tabId: String,
         val authType: AuthFollowUpType,

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt

@@ -444,7 +444,7 @@ class TelemetryHelper(private val project: Project, private val sessionStorage:
 
 data class FeedbackComment(
     val conversationId: String,
-    val messageId: String,
+    val messageId: String?,
     val reason: String,
     val userComment: String,
     val type: String = "codewhisperer-chat-answer-feedback",

plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/codeTestChatConnector.ts

@@ -4,7 +4,7 @@
 import {ExtensionMessage} from "../commands";
 import {ChatPayload, ConnectorProps} from "../connector";
 import {FormButtonIds} from "../forms/constants";
-import {ChatItem, ChatItemAction, ChatItemType, MynahIcons, MynahUIDataModel} from '@aws/mynah-ui-chat'
+import {ChatItem, ChatItemAction, ChatItemType, FeedbackPayload, MynahIcons, MynahUIDataModel} from '@aws/mynah-ui-chat'
 import {CodeReference} from "./amazonqCommonsConnector";
 import {Status} from "@aws/mynah-ui-chat/dist/static";
 import {EmptyMynahUIDataModel} from "@aws/mynah-ui-chat/dist/helper/store";
@@ -13,6 +13,7 @@ import {doesNotMatch} from "node:assert";
 export interface ICodeTestChatConnectorProps {
     sendMessageToExtension: (message: ExtensionMessage) => void
     onChatAnswerReceived?: (tabID: string, message: ChatItem) => void
+    sendFeedback?: (tabId: string, feedbackPayload: FeedbackPayload) => void | undefined
     onUpdateAuthentication: (
         featureDevEnabled: boolean,
         codeTransformEnabled: boolean,
@@ -571,6 +572,25 @@ export class CodeTestChatConnector {
         })
     }
 
+    onChatItemVoted = (tabId: string, messageId: string, vote: string): void | undefined => {
+        this.sendMessageToExtension({
+            tabID: tabId,
+            messageId: messageId,
+            vote: vote,
+            command: 'chat-item-voted',
+            tabType: 'codetest',
+        })
+    }
+
+    sendFeedback = (tabId: string, feedbackPayload: FeedbackPayload): void | undefined => {
+        this.sendMessageToExtension({
+            command: 'chat-item-feedback',
+            ...feedbackPayload,
+            tabType: 'codetest',
+            tabID: tabId,
+        })
+    }
+
     onResponseBodyLinkClick = (tabID: string, messageId: string, link: string): void => {
         this.sendMessageToExtension({
             command: 'response-body-link-click',

plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts

@@ -593,6 +593,9 @@ export class Connector {
             case 'cwc':
                 this.cwChatConnector.onSendFeedback(tabId, feedbackPayload)
                 break
+            case 'codetest':
+                this.codeTestChatConnector.sendFeedback(tabId,feedbackPayload)
+                break
         }
     }
 
@@ -604,6 +607,9 @@ export class Connector {
             case 'featuredev':
                 this.featureDevChatConnector.onChatItemVoted(tabId, messageId, vote)
                 break
+            case 'codetest' :
+                this.codeTestChatConnector.onChatItemVoted(tabId,messageId,vote)
+                break
         }
     }
 }