telemetry(amazonq): sending metric data in onCodeGeneration

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt

@@ -38,3 +38,21 @@ enum class FeatureDevOperation(private val operationName: String) {
 
     override fun toString(): String = operationName
 }
+
+enum class MetricDataOperationName(private val operationName: String) {
+    StartCodeGeneration("StartCodeGeneration"),
+    EndCodeGeneration("EndCodeGeneration"),
+    ;
+
+    override fun toString(): String = operationName
+}
+
+enum class MetricDataResult(private val resultName: String) {
+    Success("Success"),
+    Fault("Fault"),
+    Error("Error"),
+    LlmFailure("LLMFailure"),
+    ;
+
+    override fun toString(): String = resultName
+}

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt

@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
 import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
@@ -89,6 +90,33 @@ class FeatureDevClient(
             requestBuilder.userContext(featureDevUserContext)
         }
 
+    fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse =
+        bearerClient().sendTelemetryEvent { requestBuilder ->
+            requestBuilder.telemetryEvent { telemetryEventBuilder ->
+                telemetryEventBuilder.metricData {
+                    it
+                        .metricName("Operation")
+                        .metricValue(1.0)
+                        .timestamp(Instant.now())
+                        .product("FeatureDev")
+                        .dimensions(
+                            listOf(
+                                Dimension.builder()
+                                    .name("operationName")
+                                    .value(operationName)
+                                    .build(),
+                                Dimension.builder()
+                                    .name("result")
+                                    .value(result)
+                                    .build()
+                            )
+                        )
+                }
+            }
+            requestBuilder.optOutPreference(getTelemetryOptOutPreference())
+            requestBuilder.userContext(featureDevUserContext)
+        }
+
     fun sendFeatureDevCodeGenerationEvent(
         conversationId: String,
         linesOfCodeGenerated: Int,

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt

@@ -6,6 +6,13 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller
 import com.intellij.notification.NotificationAction
 import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType
@@ -63,6 +70,11 @@ suspend fun FeatureDevController.onCodeGeneration(
 
         messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code"))
 
+        session.sendMetricDataTelemetry(
+            MetricDataOperationName.StartCodeGeneration,
+            MetricDataResult.Success
+        )
+
         session.send(message) // Trigger code generation
 
         state = session.sessionState
@@ -134,8 +146,29 @@ suspend fun FeatureDevController.onCodeGeneration(
         }
 
         messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL))
-
         messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
+    } catch (err: Exception) {
+        when (err) {
+            is GuardrailsException, is NoChangeRequiredException, is PromptRefusalException, is ThrottlingException -> {
+                session.sendMetricDataTelemetry(
+                    MetricDataOperationName.EndCodeGeneration,
+                    MetricDataResult.Error
+                )
+            }
+            is EmptyPatchException -> {
+                session.sendMetricDataTelemetry(
+                    MetricDataOperationName.EndCodeGeneration,
+                    MetricDataResult.LlmFailure
+                )
+            }
+            else -> {
+                session.sendMetricDataTelemetry(
+                    MetricDataOperationName.EndCodeGeneration,
+                    MetricDataResult.Fault
+                )
+            }
+        }
+        throw err
     } finally {
         if (session.sessionState.token
                 ?.token
@@ -155,6 +188,11 @@ suspend fun FeatureDevController.onCodeGeneration(
             )
         }
     }
+
+    session.sendMetricDataTelemetry(
+        MetricDataOperationName.EndCodeGeneration,
+        MetricDataResult.Success
+    )
 }
 
 private suspend fun disposeToken(

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt

@@ -14,6 +14,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATIO
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ConversationIdNotFoundException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_NAME
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MAX_PROJECT_SIZE_BYTES
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.IncomingFeatureDevMessage
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendAsyncEventProgress
@@ -219,6 +221,10 @@ class Session(val tabID: String, val project: Project) {
         this._codeResultMessageId = null
     }
 
+    fun sendMetricDataTelemetry(operationName: MetricDataOperationName, result: MetricDataResult) {
+        featureDevService.sendFeatureDevMetricData(operationName.toString(), result.toString())
+    }
+
     suspend fun send(msg: String): Interaction {
         // When the task/"thing to do" hasn't been set yet, we want it to be the incoming message
         if (task.isEmpty() && msg.isNotEmpty()) {

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt

@@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project)
         }
     }
 
+    fun sendFeatureDevMetricData(operationName: String, result: String) {
+        val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
+        try {
+            sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result)
+            val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId()
+            logger.debug {
+                "$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId"
+            }
+        } catch (e: Exception) {
+            logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" }
+        }
+    }
+
     fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) {
         val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
         try {

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt

@@ -22,9 +22,11 @@ import org.junit.After
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
+import org.junit.jupiter.api.assertThrows
 import org.mockito.kotlin.any
 import org.mockito.kotlin.doNothing
 import org.mockito.kotlin.doReturn
+import org.mockito.kotlin.inOrder
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.reset
 import org.mockito.kotlin.spy
@@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
 import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
 import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates
 import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
@@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction
@@ -423,6 +433,114 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
             }
         }
 
+    @Test
+    fun `test handleChat onCodeGeneration sends success metrics`() = runTest {
+        val mockSession = mock<Session>()
+        val featureDevService = mockk<FeatureDevService>()
+        val repoContext = mock<FeatureDevSessionContext>()
+        val sessionStateConfig = SessionStateConfig(testConversationId, repoContext, featureDevService)
+        val mockInteraction = mock<Interaction>()
+        whenever(mockSession.send(userMessage)).thenReturn(mockInteraction)
+        whenever(mockSession.sessionState).thenReturn(
+            PrepareCodeGenerationState(
+                testTabId,
+                CancellationTokenSource(),
+                "test-command",
+                sessionStateConfig,
+                newFileContents,
+                deletedFiles,
+                testReferences,
+                testUploadId,
+                0,
+                messenger,
+                diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()),
+            ),
+        )
+
+        controller.onCodeGeneration(mockSession, userMessage, testTabId)
+
+        val mockInOrder = inOrder(mockSession)
+
+        mockInOrder.verify(mockSession).sendMetricDataTelemetry(
+            MetricDataOperationName.StartCodeGeneration,
+            MetricDataResult.Success
+
+        )
+        mockInOrder.verify(mockSession).sendMetricDataTelemetry(
+            MetricDataOperationName.EndCodeGeneration,
+            MetricDataResult.Success
+        )
+    }
+
+    @Test
+    fun `test handleChat onCodeGeneration sends correct failure metrics for different errors`() = runTest {
+        data class ErrorTestCase(
+            val error: Exception,
+            val expectedMetricResult: MetricDataResult,
+        )
+
+        val testCases = listOf(
+            ErrorTestCase(
+                EmptyPatchException("EmptyPatchException", "Empty patch"),
+                MetricDataResult.LlmFailure
+            ),
+            ErrorTestCase(
+                GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"),
+                MetricDataResult.Error
+            ),
+            ErrorTestCase(
+                PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"),
+                MetricDataResult.Error
+            ),
+            ErrorTestCase(
+                NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"),
+                MetricDataResult.Error
+            ),
+            ErrorTestCase(
+                ThrottlingException(operation = "GenerateCode", desc = "Request throttled"),
+                MetricDataResult.Error
+            ),
+            ErrorTestCase(
+                RuntimeException("Unknown error"),
+                MetricDataResult.Fault
+            )
+        )
+
+        testCases.forEach { (error, expectedResult) ->
+            val mockSession = mock<Session>()
+            whenever(mockSession.send(userMessage)).thenThrow(error)
+            whenever(mockSession.sessionState).thenReturn(
+                CodeGenerationState(
+                    testTabId,
+                    "",
+                    mock(),
+                    testUploadId,
+                    0,
+                    0.0,
+                    messenger,
+                    token = CancellationTokenSource(),
+                    diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet())
+                )
+            )
+
+            assertThrows<Exception> {
+                controller.onCodeGeneration(mockSession, userMessage, testTabId)
+            }
+
+            val mockInOrder = inOrder(mockSession)
+
+            mockInOrder.verify(mockSession).sendMetricDataTelemetry(
+                MetricDataOperationName.StartCodeGeneration,
+                MetricDataResult.Success
+
+            )
+            mockInOrder.verify(mockSession).sendMetricDataTelemetry(
+                MetricDataOperationName.EndCodeGeneration,
+                expectedResult
+            )
+        }
+    }
+
     @Test
     fun `test processFileClicked handles file rejection`() =
         runTest {