telemetry(/dev): add feedback telemetry (#4867)

chengoramazon · web-flow · commit 54f77f603f8f · 2024-09-10T11:19:07.000-07:00
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevApp.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevApp.kt
@@ -37,6 +37,7 @@ class FeatureDevApp : AmazonQApp {
             "auth-follow-up-was-clicked" to IncomingFeatureDevMessage.AuthFollowUpWasClicked::class,
             "follow-up-was-clicked" to IncomingFeatureDevMessage.FollowupClicked::class,
             "chat-item-voted" to IncomingFeatureDevMessage.ChatItemVotedMessage::class,
+            "chat-item-feedback" to IncomingFeatureDevMessage.ChatItemFeedbackMessage::class,
             "response-body-link-click" to IncomingFeatureDevMessage.ClickedLink::class,
             "insert_code_at_cursor_position" to IncomingFeatureDevMessage.InsertCodeAtCursorPosition::class,
             "open-diff" to IncomingFeatureDevMessage.OpenDiff::class,
@@ -76,6 +77,7 @@ class FeatureDevApp : AmazonQApp {
             is IncomingFeatureDevMessage.AuthFollowUpWasClicked -> inboundAppMessagesHandler.processAuthFollowUpClick(message)
             is IncomingFeatureDevMessage.FollowupClicked -> inboundAppMessagesHandler.processFollowupClickedMessage(message)
             is IncomingFeatureDevMessage.ChatItemVotedMessage -> inboundAppMessagesHandler.processChatItemVotedMessage(message)
+            is IncomingFeatureDevMessage.ChatItemFeedbackMessage -> inboundAppMessagesHandler.processChatItemFeedbackMessage(message)
             is IncomingFeatureDevMessage.ClickedLink -> inboundAppMessagesHandler.processLinkClick(message)
             is IncomingFeatureDevMessage.InsertCodeAtCursorPosition -> inboundAppMessagesHandler.processInsertCodeAtCursorPosition(message)
             is IncomingFeatureDevMessage.OpenDiff -> inboundAppMessagesHandler.processOpenDiff(message)
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/InboundAppMessagesHandler.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/InboundAppMessagesHandler.kt
@@ -12,6 +12,7 @@ interface InboundAppMessagesHandler {
     suspend fun processAuthFollowUpClick(message: IncomingFeatureDevMessage.AuthFollowUpWasClicked)
     suspend fun processFollowupClickedMessage(message: IncomingFeatureDevMessage.FollowupClicked)
     suspend fun processChatItemVotedMessage(message: IncomingFeatureDevMessage.ChatItemVotedMessage)
+    suspend fun processChatItemFeedbackMessage(message: IncomingFeatureDevMessage.ChatItemFeedbackMessage)
     suspend fun processLinkClick(message: IncomingFeatureDevMessage.ClickedLink)
     suspend fun processInsertCodeAtCursorPosition(message: IncomingFeatureDevMessage.InsertCodeAtCursorPosition)
     suspend fun processOpenDiff(message: IncomingFeatureDevMessage.OpenDiff)
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevController.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevController.kt
@@ -3,6 +3,7 @@
 
 package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller
 
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.intellij.diff.DiffContentFactory
 import com.intellij.diff.DiffManager
 import com.intellij.diff.contents.EmptyContent
@@ -17,6 +18,7 @@ import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.wm.ToolWindowManager
 import kotlinx.coroutines.withContext
+import software.amazon.awssdk.services.toolkittelemetry.model.Sentiment
 import software.aws.toolkits.core.utils.debug
 import software.aws.toolkits.core.utils.error
 import software.aws.toolkits.core.utils.getLogger
@@ -62,8 +64,11 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Sessio
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.SessionStatePhase
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.storage.ChatSessionStorage
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.selectFolder
+import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.FeedbackComment
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
+import software.aws.toolkits.jetbrains.services.telemetry.TelemetryService
 import software.aws.toolkits.jetbrains.ui.feedback.FeatureDevFeedbackDialog
+import software.aws.toolkits.jetbrains.utils.notifyError
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.AmazonqTelemetry
 import software.aws.toolkits.telemetry.Result
@@ -132,6 +137,32 @@ class FeatureDevController(
         }
     }
 
+    override suspend fun processChatItemFeedbackMessage(message: IncomingFeatureDevMessage.ChatItemFeedbackMessage) {
+        logger.debug { "$FEATURE_NAME: Processing ChatItemFeedbackMessage: ${message.comment}" }
+
+        val session = getSessionInfo(message.tabId)
+
+        val comment = FeedbackComment(
+            conversationId = session.conversationId,
+            userComment = message.comment.orEmpty(),
+            reason = message.selectedOption,
+            messageId = message.messageId,
+            type = "featuredev-chat-answer-feedback"
+        )
+
+        try {
+            TelemetryService.getInstance().sendFeedback(
+                sentiment = Sentiment.NEGATIVE,
+                comment = objectMapper.writeValueAsString(comment),
+            )
+            logger.info { "$FEATURE_NAME answer feedback sent: \"Negative\"" }
+        } catch (e: Throwable) {
+            e.notifyError(message("feedback.submit_failed", e))
+            logger.warn(e) { "Failed to submit feedback" }
+            return
+        }
+    }
+
     override suspend fun processLinkClick(message: IncomingFeatureDevMessage.ClickedLink) {
         BrowserUtil.browse(message.link)
     }
@@ -284,7 +315,8 @@ class FeatureDevController(
             messenger.sendAnswer(
                 tabId = tabId,
                 message = message("amazonqFeatureDev.code_generation.updated_code"),
-                messageType = FeatureDevMessageType.Answer
+                messageType = FeatureDevMessageType.Answer,
+                canBeVoted = true
             )
 
             messenger.sendSystemPrompt(
@@ -331,9 +363,18 @@ class FeatureDevController(
     }
 
     private suspend fun closeSession(tabId: String) {
-        messenger.sendAnswer(tabId = tabId, messageType = FeatureDevMessageType.Answer, message = message("amazonqFeatureDev.chat_message.closed_session"))
+        messenger.sendAnswer(
+            tabId = tabId,
+            messageType = FeatureDevMessageType.Answer,
+            message = message("amazonqFeatureDev.chat_message.closed_session"),
+            canBeVoted = true
+        )
+
+        messenger.sendUpdatePlaceholder(
+            tabId = tabId,
+            newPlaceholder = message("amazonqFeatureDev.placeholder.closed_session")
+        )
 
-        messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.closed_session"))
         messenger.sendChatInputEnabledMessage(tabId = tabId, enabled = false)
 
         val session = getSessionInfo(tabId)
@@ -360,7 +401,8 @@ class FeatureDevController(
         messenger.sendAnswer(
             tabId = tabId,
             message = message("amazonqFeatureDev.code_generation.provide_code_feedback"),
-            messageType = FeatureDevMessageType.Answer
+            messageType = FeatureDevMessageType.Answer,
+            canBeVoted = true
         )
         messenger.sendUpdatePlaceholder(tabId, message("amazonqFeatureDev.placeholder.provide_code_feedback"))
     }
@@ -575,6 +617,7 @@ class FeatureDevController(
                 tabId = tabId,
                 messageType = FeatureDevMessageType.Answer,
                 message = message("amazonqFeatureDev.follow_up.modified_source_folder", selectedFolder.path),
+                canBeVoted = true,
             )
 
             messenger.sendAnswer(
@@ -615,5 +658,7 @@ class FeatureDevController(
 
     companion object {
         private val logger = getLogger<FeatureDevController>()
+
+        private val objectMapper = jacksonObjectMapper()
     }
 }
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/messages/FeatureDevMessage.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/messages/FeatureDevMessage.kt
@@ -52,6 +52,13 @@ sealed interface IncomingFeatureDevMessage : FeatureDevBaseMessage {
         val vote: String,
     ) : IncomingFeatureDevMessage
 
+    data class ChatItemFeedbackMessage(
+        @JsonProperty("tabID") val tabId: String,
+        val selectedOption: String,
+        val comment: String?,
+        val messageId: String,
+    ) : IncomingFeatureDevMessage
+
     data class ClickedLink(
         @JsonProperty("tabID") val tabId: String,
         val command: String,
diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt
@@ -157,7 +157,12 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         mockitoVerify(chatSessionStorage, times(1)).deleteSession(testTabId)
 
         coVerifyOrder {
-            messenger.sendAnswer(testTabId, message("amazonqFeatureDev.chat_message.closed_session"), FeatureDevMessageType.Answer)
+            messenger.sendAnswer(
+                tabId = testTabId,
+                message = message("amazonqFeatureDev.chat_message.closed_session"),
+                messageType = FeatureDevMessageType.Answer,
+                canBeVoted = true
+            )
             messenger.sendUpdatePlaceholder(testTabId, message("amazonqFeatureDev.placeholder.closed_session"))
             messenger.sendChatInputEnabledMessage(testTabId, false)
             messenger.sendAnswer(testTabId, message("amazonqFeatureDev.chat_message.ask_for_new_task"), FeatureDevMessageType.Answer)
@@ -187,7 +192,12 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         coVerifyOrder {
             AmazonqTelemetry.isProvideFeedbackForCodeGen(amazonqConversationId = testConversationId, enabled = true, createTime = any())
             messenger.sendAsyncEventProgress(testTabId, inProgress = false)
-            messenger.sendAnswer(testTabId, message("amazonqFeatureDev.code_generation.provide_code_feedback"), FeatureDevMessageType.Answer)
+            messenger.sendAnswer(
+                tabId = testTabId,
+                message = message("amazonqFeatureDev.code_generation.provide_code_feedback"),
+                messageType = FeatureDevMessageType.Answer,
+                canBeVoted = true
+            )
             messenger.sendUpdatePlaceholder(testTabId, message("amazonqFeatureDev.placeholder.provide_code_feedback"))
         }
     }
@@ -226,7 +236,12 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
                 enabled = true,
                 createTime = any()
             )
-            messenger.sendAnswer(testTabId, message("amazonqFeatureDev.code_generation.updated_code"), FeatureDevMessageType.Answer)
+            messenger.sendAnswer(
+                tabId = testTabId,
+                message = message("amazonqFeatureDev.code_generation.updated_code"),
+                messageType = FeatureDevMessageType.Answer,
+                canBeVoted = true
+            )
             messenger.sendSystemPrompt(
                 testTabId,
                 listOf(
@@ -441,7 +456,8 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
             messenger.sendAnswer(
                 tabId = testTabId,
                 messageType = FeatureDevMessageType.Answer,
-                message = message("amazonqFeatureDev.follow_up.modified_source_folder", folder.path)
+                message = message("amazonqFeatureDev.follow_up.modified_source_folder", folder.path),
+                canBeVoted = true,
             )
         }
     }
diff --git a/plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/featureDevChatConnector.ts b/plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/featureDevChatConnector.ts
@@ -280,7 +280,12 @@ export class Connector {
     }
 
     sendFeedback = (tabId: string, feedbackPayload: FeedbackPayload): void | undefined => {
-        // TODO implement telemetry
+        this.sendMessageToExtension({
+            command: 'chat-item-feedback',
+            ...feedbackPayload,
+            tabType: 'featuredev',
+            tabID: tabId,
+        })
     }
 
     onChatItemVoted = (tabId: string, messageId: string, vote: string): void | undefined => {

Original file line number	Diff line number	Diff line change
`@@ -280,7 +280,12 @@ export class Connector {`
`280`	`280`	`}`
`281`	`281`
`282`	`282`	`sendFeedback = (tabId: string, feedbackPayload: FeedbackPayload): void \| undefined => {`
`283`		`- // TODO implement telemetry`
	`283`	`+ this.sendMessageToExtension({`
	`284`	`+ command: 'chat-item-feedback',`
	`285`	`+ ...feedbackPayload,`
	`286`	`+ tabType: 'featuredev',`
	`287`	`+ tabID: tabId,`
	`288`	`+ })`
`284`	`289`	`}`
`285`	`290`
`286`	`291`	`onChatItemVoted = (tabId: string, messageId: string, vote: string): void \| undefined => {`