From a16b696573fce65cdcd050e6bbfe6f5278c362a8 Mon Sep 17 00:00:00 2001 From: Zhuowen Chen Date: Fri, 13 Dec 2024 14:24:45 -0800 Subject: [PATCH 1/5] feat(amazonq): sending metric data in onCodeGeneration --- .../amazonqFeatureDev/FeatureDevConstants.kt | 18 +++++ .../clients/FeatureDevClient.kt | 28 +++++++ .../FeatureDevControllerExtensions.kt | 63 +++++++++++++++ .../amazonqFeatureDev/session/Session.kt | 4 + .../util/FeatureDevService.kt | 13 +++ .../controller/FeatureDevControllerTest.kt | 79 +++++++++++++++++++ 6 files changed, 205 insertions(+) diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt index b8678aead68..af0b724d967 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt @@ -38,3 +38,21 @@ enum class FeatureDevOperation(private val operationName: String) { override fun toString(): String = operationName } + +enum class MetricDataOperationName(private val operationName: String) { + START_CODE_GENERATION("StartCodeGeneration"), + END_CODE_GENERATION("EndCodeGeneration"), + ; + + override fun toString(): String = operationName +} + +enum class MetricDataResult(private val resultName: String) { + SUCCESS("Success"), + FAULT("Fault"), + ERROR("Error"), + LLMFAILURE("LLMFailure"), + ; + + override fun toString(): String = resultName +} diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt index 29a1feb1a05..c05d59ef50f 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse +import software.amazon.awssdk.services.codewhispererruntime.model.Dimension import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem @@ -89,6 +90,33 @@ class FeatureDevClient( requestBuilder.userContext(featureDevUserContext) } + fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse = + bearerClient().sendTelemetryEvent { requestBuilder -> + requestBuilder.telemetryEvent { telemetryEventBuilder -> + telemetryEventBuilder.metricData { + it + .metricName("Operation") + .metricValue(1.0) + .timestamp(Instant.now()) + .product("Amazon Q For JetBrains") + .dimensions( + listOf( + Dimension.builder() + .name("operationName") + .value(operationName) + .build(), + Dimension.builder() + .name("result") + .value(result) + .build() + ) + ) + } + } + requestBuilder.optOutPreference(getTelemetryOptOutPreference()) + requestBuilder.userContext(featureDevUserContext) + } + fun sendFeatureDevCodeGenerationEvent( conversationId: String, linesOfCodeGenerated: Int, diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt index 19331d7f611..68e5cfe7c45 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt @@ -6,6 +6,14 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller import com.intellij.notification.NotificationAction import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType @@ -63,6 +71,11 @@ suspend fun FeatureDevController.onCodeGeneration( messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code")) + session.sendMetricDataTelemetry( + MetricDataOperationName.START_CODE_GENERATION.toString(), + MetricDataResult.SUCCESS.toString() + ) + session.send(message) // Trigger code generation state = session.sessionState @@ -135,7 +148,57 @@ suspend fun FeatureDevController.onCodeGeneration( messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL)) + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.SUCCESS.toString() + ) messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation")) + } catch (err: FeatureDevException) { + when (true) { + (err is GuardrailsException) -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.ERROR.toString() + ) + } + (err is PromptRefusalException) -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.ERROR.toString() + ) + } + (err is EmptyPatchException) -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.LLMFAILURE.toString() + ) + } + (err is NoChangeRequiredException) -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.ERROR.toString() + ) + } + (err is ThrottlingException) -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.ERROR.toString() + ) + } + else -> { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.FAULT.toString() + ) + } + } + throw err + } catch (err: Exception) { + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.FAULT.toString() + ) + throw err } finally { if (session.sessionState.token ?.token diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt index cbf2873c224..af70d7fbcf1 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt @@ -219,6 +219,10 @@ class Session(val tabID: String, val project: Project) { this._codeResultMessageId = null } + fun sendMetricDataTelemetry(operationName: String, result: String) { + featureDevService.sendFeatureDevMetricData(operationName, result) + } + suspend fun send(msg: String): Interaction { // When the task/"thing to do" hasn't been set yet, we want it to be the incoming message if (task.isEmpty() && msg.isNotEmpty()) { diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt index 123105e850a..5a8e011aa62 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt @@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project) } } + fun sendFeatureDevMetricData(operationName: String, result: String) { + val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse + try { + sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result) + val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId() + logger.debug { + "$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId" + } + } catch (e: Exception) { + logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" } + } + } + fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) { val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse try { diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt index 4cd4077e828..5d16c609590 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt @@ -22,9 +22,11 @@ import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import org.junit.jupiter.api.assertThrows import org.mockito.kotlin.any import org.mockito.kotlin.doNothing import org.mockito.kotlin.doReturn +import org.mockito.kotlin.inOrder import org.mockito.kotlin.mock import org.mockito.kotlin.reset import org.mockito.kotlin.spy @@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp @@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction @@ -423,6 +433,75 @@ class FeatureDevControllerTest : FeatureDevTestBase() { } } + @Test + fun `test handleChat onCodeGeneration sends correct metrics for different errors`() = runTest { + data class ErrorTestCase( + val error: Exception, + val expectedMetricResult: MetricDataResult, + ) + + val testCases = listOf( + ErrorTestCase( + EmptyPatchException("EmptyPatchException", "Empty patch"), + MetricDataResult.LLMFAILURE + ), + ErrorTestCase( + GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"), + MetricDataResult.ERROR + ), + ErrorTestCase( + PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"), + MetricDataResult.ERROR + ), + ErrorTestCase( + NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"), + MetricDataResult.ERROR + ), + ErrorTestCase( + ThrottlingException(operation = "GenerateCode", desc = "Request throttled"), + MetricDataResult.ERROR + ), + ErrorTestCase( + RuntimeException("Unknown error"), + MetricDataResult.FAULT + ) + ) + + testCases.forEach { (error, expectedResult) -> + val mockSession = mock() + whenever(mockSession.send(userMessage)).thenThrow(error) + whenever(mockSession.sessionState).thenReturn( + CodeGenerationState( + testTabId, + "", + mock(), + testUploadId, + 0, + 0.0, + messenger, + token = CancellationTokenSource(), + diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()) + ) + ) + + assertThrows { + controller.onCodeGeneration(mockSession, userMessage, testTabId) + } + + val mockInOrder = inOrder(mockSession) + + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.START_CODE_GENERATION.toString(), + MetricDataResult.SUCCESS.toString() + + ) + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + expectedResult.toString() + ) + } + } + @Test fun `test processFileClicked handles file rejection`() = runTest { From a549c4aba8749a6bbbf40760e1ae693781040ecc Mon Sep 17 00:00:00 2001 From: Zhuowen Chen Date: Tue, 17 Dec 2024 14:42:00 -0800 Subject: [PATCH 2/5] Add happy path --- .../clients/FeatureDevClient.kt | 2 +- .../controller/FeatureDevControllerTest.kt | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt index c05d59ef50f..2d73f5fb4ba 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt @@ -98,7 +98,7 @@ class FeatureDevClient( .metricName("Operation") .metricValue(1.0) .timestamp(Instant.now()) - .product("Amazon Q For JetBrains") + .product("FeatureDev") .dimensions( listOf( Dimension.builder() diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt index 5d16c609590..9e57b7cf107 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt @@ -434,7 +434,46 @@ class FeatureDevControllerTest : FeatureDevTestBase() { } @Test - fun `test handleChat onCodeGeneration sends correct metrics for different errors`() = runTest { + fun `test handleChat onCodeGeneration sends success metrics`() = runTest { + val mockSession = mock() + val featureDevService = mockk() + val repoContext = mock() + val sessionStateConfig = SessionStateConfig(testConversationId, repoContext, featureDevService) + val mockInteraction = mock() + whenever(mockSession.send(userMessage)).thenReturn(mockInteraction) + whenever(mockSession.sessionState).thenReturn( + PrepareCodeGenerationState( + testTabId, + CancellationTokenSource(), + "test-command", + sessionStateConfig, + newFileContents, + deletedFiles, + testReferences, + testUploadId, + 0, + messenger, + diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()), + ), + ) + + controller.onCodeGeneration(mockSession, userMessage, testTabId) + + val mockInOrder = inOrder(mockSession) + + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.START_CODE_GENERATION.toString(), + MetricDataResult.SUCCESS.toString() + + ) + mockInOrder.verify(mockSession).sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION.toString(), + MetricDataResult.SUCCESS.toString() + ) + } + + @Test + fun `test handleChat onCodeGeneration sends correct failure metrics for different errors`() = runTest { data class ErrorTestCase( val error: Exception, val expectedMetricResult: MetricDataResult, From 02441f2622248c99fc7082ca9e8b52090a592f58 Mon Sep 17 00:00:00 2001 From: Zhuowen Chen Date: Tue, 17 Dec 2024 16:19:35 -0800 Subject: [PATCH 3/5] Fix comment --- .../FeatureDevControllerExtensions.kt | 42 +++++++++---------- .../amazonqFeatureDev/session/Session.kt | 6 ++- .../controller/FeatureDevControllerTest.kt | 16 +++---- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt index 68e5cfe7c45..61050ecaa67 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt @@ -72,8 +72,8 @@ suspend fun FeatureDevController.onCodeGeneration( messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code")) session.sendMetricDataTelemetry( - MetricDataOperationName.START_CODE_GENERATION.toString(), - MetricDataResult.SUCCESS.toString() + MetricDataOperationName.START_CODE_GENERATION, + MetricDataResult.SUCCESS ) session.send(message) // Trigger code generation @@ -147,56 +147,51 @@ suspend fun FeatureDevController.onCodeGeneration( } messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL)) - - session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.SUCCESS.toString() - ) messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation")) } catch (err: FeatureDevException) { when (true) { (err is GuardrailsException) -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.ERROR.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.ERROR ) } (err is PromptRefusalException) -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.ERROR.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.ERROR ) } (err is EmptyPatchException) -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.LLMFAILURE.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.LLMFAILURE ) } (err is NoChangeRequiredException) -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.ERROR.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.ERROR ) } (err is ThrottlingException) -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.ERROR.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.ERROR ) } else -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.FAULT.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.FAULT ) } } throw err } catch (err: Exception) { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.FAULT.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.FAULT ) throw err } finally { @@ -218,6 +213,11 @@ suspend fun FeatureDevController.onCodeGeneration( ) } } + + session.sendMetricDataTelemetry( + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.SUCCESS + ) } private suspend fun disposeToken( diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt index af70d7fbcf1..3f432ba8c7f 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt @@ -14,6 +14,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATIO import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ConversationIdNotFoundException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_NAME import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MAX_PROJECT_SIZE_BYTES +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName +import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.IncomingFeatureDevMessage import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendAsyncEventProgress @@ -219,8 +221,8 @@ class Session(val tabID: String, val project: Project) { this._codeResultMessageId = null } - fun sendMetricDataTelemetry(operationName: String, result: String) { - featureDevService.sendFeatureDevMetricData(operationName, result) + fun sendMetricDataTelemetry(operationName: MetricDataOperationName, result: MetricDataResult) { + featureDevService.sendFeatureDevMetricData(operationName.toString(), result.toString()) } suspend fun send(msg: String): Interaction { diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt index 9e57b7cf107..f1710b2b74d 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt @@ -462,13 +462,13 @@ class FeatureDevControllerTest : FeatureDevTestBase() { val mockInOrder = inOrder(mockSession) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.START_CODE_GENERATION.toString(), - MetricDataResult.SUCCESS.toString() + MetricDataOperationName.START_CODE_GENERATION, + MetricDataResult.SUCCESS ) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - MetricDataResult.SUCCESS.toString() + MetricDataOperationName.END_CODE_GENERATION, + MetricDataResult.SUCCESS ) } @@ -530,13 +530,13 @@ class FeatureDevControllerTest : FeatureDevTestBase() { val mockInOrder = inOrder(mockSession) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.START_CODE_GENERATION.toString(), - MetricDataResult.SUCCESS.toString() + MetricDataOperationName.START_CODE_GENERATION, + MetricDataResult.SUCCESS ) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION.toString(), - expectedResult.toString() + MetricDataOperationName.END_CODE_GENERATION, + expectedResult ) } } From 519d66639b5d8cbdf8377aaeeeca48bfc2342218 Mon Sep 17 00:00:00 2001 From: Zhuowen Chen Date: Wed, 18 Dec 2024 14:07:54 -0800 Subject: [PATCH 4/5] Fix review --- .../amazonqFeatureDev/FeatureDevConstants.kt | 12 ++--- .../FeatureDevControllerExtensions.kt | 51 ++++++------------- .../controller/FeatureDevControllerTest.kt | 26 +++++----- 3 files changed, 35 insertions(+), 54 deletions(-) diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt index af0b724d967..2f0b06f1b2f 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt @@ -40,18 +40,18 @@ enum class FeatureDevOperation(private val operationName: String) { } enum class MetricDataOperationName(private val operationName: String) { - START_CODE_GENERATION("StartCodeGeneration"), - END_CODE_GENERATION("EndCodeGeneration"), + StartCodeGeneration("StartCodeGeneration"), + EndCodeGeneration("EndCodeGeneration"), ; override fun toString(): String = operationName } enum class MetricDataResult(private val resultName: String) { - SUCCESS("Success"), - FAULT("Fault"), - ERROR("Error"), - LLMFAILURE("LLMFailure"), + Success("Success"), + Fault("Fault"), + Error("Error"), + LlmFailure("LLMFailure"), ; override fun toString(): String = resultName diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt index 61050ecaa67..e2e970dd1c8 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt @@ -7,7 +7,6 @@ import com.intellij.notification.NotificationAction import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException -import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult @@ -72,8 +71,8 @@ suspend fun FeatureDevController.onCodeGeneration( messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code")) session.sendMetricDataTelemetry( - MetricDataOperationName.START_CODE_GENERATION, - MetricDataResult.SUCCESS + MetricDataOperationName.StartCodeGeneration, + MetricDataResult.Success ) session.send(message) // Trigger code generation @@ -148,50 +147,32 @@ suspend fun FeatureDevController.onCodeGeneration( messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL)) messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation")) - } catch (err: FeatureDevException) { - when (true) { - (err is GuardrailsException) -> { - session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.ERROR - ) - } - (err is PromptRefusalException) -> { - session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.ERROR - ) - } - (err is EmptyPatchException) -> { - session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.LLMFAILURE - ) - } - (err is NoChangeRequiredException) -> { + } catch (err: Exception) { + when (err) { + is GuardrailsException, is NoChangeRequiredException, is PromptRefusalException, is ThrottlingException -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.ERROR + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Error ) } - (err is ThrottlingException) -> { + is EmptyPatchException -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.ERROR + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.LlmFailure ) } else -> { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.FAULT + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Fault ) } } throw err } catch (err: Exception) { session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.FAULT + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Fault ) throw err } finally { @@ -215,8 +196,8 @@ suspend fun FeatureDevController.onCodeGeneration( } session.sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.SUCCESS + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Success ) } diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt index f1710b2b74d..10e79dc58f0 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt @@ -462,13 +462,13 @@ class FeatureDevControllerTest : FeatureDevTestBase() { val mockInOrder = inOrder(mockSession) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.START_CODE_GENERATION, - MetricDataResult.SUCCESS + MetricDataOperationName.StartCodeGeneration, + MetricDataResult.Success ) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, - MetricDataResult.SUCCESS + MetricDataOperationName.EndCodeGeneration, + MetricDataResult.Success ) } @@ -482,27 +482,27 @@ class FeatureDevControllerTest : FeatureDevTestBase() { val testCases = listOf( ErrorTestCase( EmptyPatchException("EmptyPatchException", "Empty patch"), - MetricDataResult.LLMFAILURE + MetricDataResult.LlmFailure ), ErrorTestCase( GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"), - MetricDataResult.ERROR + MetricDataResult.Error ), ErrorTestCase( PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"), - MetricDataResult.ERROR + MetricDataResult.Error ), ErrorTestCase( NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"), - MetricDataResult.ERROR + MetricDataResult.Error ), ErrorTestCase( ThrottlingException(operation = "GenerateCode", desc = "Request throttled"), - MetricDataResult.ERROR + MetricDataResult.Error ), ErrorTestCase( RuntimeException("Unknown error"), - MetricDataResult.FAULT + MetricDataResult.Fault ) ) @@ -530,12 +530,12 @@ class FeatureDevControllerTest : FeatureDevTestBase() { val mockInOrder = inOrder(mockSession) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.START_CODE_GENERATION, - MetricDataResult.SUCCESS + MetricDataOperationName.StartCodeGeneration, + MetricDataResult.Success ) mockInOrder.verify(mockSession).sendMetricDataTelemetry( - MetricDataOperationName.END_CODE_GENERATION, + MetricDataOperationName.EndCodeGeneration, expectedResult ) } From 7eb12212c3af99767733f01185ee8d0fb0f07bb2 Mon Sep 17 00:00:00 2001 From: Zhuowen Chen Date: Wed, 18 Dec 2024 16:04:07 -0800 Subject: [PATCH 5/5] Fix detekt --- .../controller/FeatureDevControllerExtensions.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt index e2e970dd1c8..a80e8bec811 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt @@ -169,12 +169,6 @@ suspend fun FeatureDevController.onCodeGeneration( } } throw err - } catch (err: Exception) { - session.sendMetricDataTelemetry( - MetricDataOperationName.EndCodeGeneration, - MetricDataResult.Fault - ) - throw err } finally { if (session.sessionState.token ?.token