diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt index b8678aead68..2f0b06f1b2f 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt @@ -38,3 +38,21 @@ enum class FeatureDevOperation(private val operationName: String) { override fun toString(): String = operationName } + +enum class MetricDataOperationName(private val operationName: String) { + StartCodeGeneration("StartCodeGeneration"), + EndCodeGeneration("EndCodeGeneration"), + ; + + override fun toString(): String = operationName +} + +enum class MetricDataResult(private val resultName: String) { + Success("Success"), + Fault("Fault"), + Error("Error"), + LlmFailure("LLMFailure"), + ; + + override fun toString(): String = resultName +} diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt index 29a1feb1a05..2d73f5fb4ba 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse +import software.amazon.awssdk.services.codewhispererruntime.model.Dimension import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem @@ -89,6 +90,33 @@ class FeatureDevClient( requestBuilder.userContext(featureDevUserContext) } + fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse = + bearerClient().sendTelemetryEvent { requestBuilder -> + requestBuilder.telemetryEvent { telemetryEventBuilder -> + telemetryEventBuilder.metricData { + it + .metricName("Operation") + .metricValue(1.0) + .timestamp(Instant.now()) + .product("FeatureDev") + .dimensions( + listOf( + Dimension.builder() + .name("operationName") + .value(operationName) + .build(), + Dimension.builder() + .name("result") + .value(result) + .build() + ) + ) + } + } + requestBuilder.optOutPreference(getTelemetryOptOutPreference()) + requestBuilder.userContext(featureDevUserContext) + } + fun sendFeatureDevCodeGenerationEvent( conversationId: String, linesOfCodeGenerated: Int, diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt index 19331d7f611..a80e8bec811 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt @@ -6,6 +6,13 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller import com.intellij.notification.NotificationAction import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType @@ -63,6 +70,11 @@ suspend fun FeatureDevController.onCodeGeneration( messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code")) + session.sendMetricDataTelemetry( + MetricDataOperationName.StartCodeGeneration, + MetricDataResult.Success + ) + session.send(message) // Trigger code generation state = session.sessionState @@ -134,8 +146,29 @@ suspend fun FeatureDevController.onCodeGeneration( } messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL)) - messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation")) + } catch (err: Exception) { + when (err) { + is GuardrailsException, is NoChangeRequiredException, is PromptRefusalException, is ThrottlingException -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Error + ) + } + is EmptyPatchException -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.LlmFailure + ) + } + else -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Fault + ) + } + } + throw err } finally { if (session.sessionState.token ?.token @@ -155,6 +188,11 @@ suspend fun FeatureDevController.onCodeGeneration( ) } } + + session.sendMetricDataTelemetry( + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Success + ) } private suspend fun disposeToken( diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt index cbf2873c224..3f432ba8c7f 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt @@ -14,6 +14,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATIO import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ConversationIdNotFoundException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_NAME import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MAX_PROJECT_SIZE_BYTES +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.IncomingFeatureDevMessage import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendAsyncEventProgress @@ -219,6 +221,10 @@ class Session(val tabID: String, val project: Project) { this._codeResultMessageId = null } + fun sendMetricDataTelemetry(operationName: MetricDataOperationName, result: MetricDataResult) { + featureDevService.sendFeatureDevMetricData(operationName.toString(), result.toString()) + } + suspend fun send(msg: String): Interaction { // When the task/"thing to do" hasn't been set yet, we want it to be the incoming message if (task.isEmpty() && msg.isNotEmpty()) { diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt index 123105e850a..5a8e011aa62 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt @@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project) } } + fun sendFeatureDevMetricData(operationName: String, result: String) { + val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse + try { + sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result) + val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId() + logger.debug { + "$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId" + } + } catch (e: Exception) { + logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" } + } + } + fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) { val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse try { diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt index 4cd4077e828..10e79dc58f0 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt @@ -22,9 +22,11 @@ import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import org.junit.jupiter.api.assertThrows import org.mockito.kotlin.any import org.mockito.kotlin.doNothing import org.mockito.kotlin.doReturn +import org.mockito.kotlin.inOrder import org.mockito.kotlin.mock import org.mockito.kotlin.reset import org.mockito.kotlin.spy @@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp @@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction @@ -423,6 +433,114 @@ class FeatureDevControllerTest : FeatureDevTestBase() { } } + @Test + fun `test handleChat onCodeGeneration sends success metrics`() = runTest { + val mockSession = mock() + val featureDevService = mockk() + val repoContext = mock() + val sessionStateConfig = SessionStateConfig(testConversationId, repoContext, featureDevService) + val mockInteraction = mock() + whenever(mockSession.send(userMessage)).thenReturn(mockInteraction) + whenever(mockSession.sessionState).thenReturn( + PrepareCodeGenerationState( + testTabId, + CancellationTokenSource(), + "test-command", + sessionStateConfig, + newFileContents, + deletedFiles, + testReferences, + testUploadId, + 0, + messenger, + diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()), + ), + ) + + controller.onCodeGeneration(mockSession, userMessage, testTabId) + + val mockInOrder = inOrder(mockSession) + + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.StartCodeGeneration, + MetricDataResult.Success + + ) + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Success + ) + } + + @Test + fun `test handleChat onCodeGeneration sends correct failure metrics for different errors`() = runTest { + data class ErrorTestCase( + val error: Exception, + val expectedMetricResult: MetricDataResult, + ) + + val testCases = listOf( + ErrorTestCase( + EmptyPatchException("EmptyPatchException", "Empty patch"), + MetricDataResult.LlmFailure + ), + ErrorTestCase( + GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"), + MetricDataResult.Error + ), + ErrorTestCase( + PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"), + MetricDataResult.Error + ), + ErrorTestCase( + NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"), + MetricDataResult.Error + ), + ErrorTestCase( + ThrottlingException(operation = "GenerateCode", desc = "Request throttled"), + MetricDataResult.Error + ), + ErrorTestCase( + RuntimeException("Unknown error"), + MetricDataResult.Fault + ) + ) + + testCases.forEach { (error, expectedResult) -> + val mockSession = mock() + whenever(mockSession.send(userMessage)).thenThrow(error) + whenever(mockSession.sessionState).thenReturn( + CodeGenerationState( + testTabId, + "", + mock(), + testUploadId, + 0, + 0.0, + messenger, + token = CancellationTokenSource(), + diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()) + ) + ) + + assertThrows { + controller.onCodeGeneration(mockSession, userMessage, testTabId) + } + + val mockInOrder = inOrder(mockSession) + + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.StartCodeGeneration, + MetricDataResult.Success + + ) + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.EndCodeGeneration, + expectedResult + ) + } + } + @Test fun `test processFileClicked handles file rejection`() = runTest {