Skip to content

Commit 846d57f

Browse files
siakmun-awssiakmun-aws
committed
feat(amazonq): sending metric data in onCodeGeneration
1 parent f0cd876 commit 846d57f

File tree

6 files changed

+205
-0
lines changed

6 files changed

+205
-0
lines changed

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,21 @@ enum class FeatureDevOperation(private val operationName: String) {
3838

3939
override fun toString(): String = operationName
4040
}
41+
42+
enum class MetricDataOperationName(private val operationName: String) {
43+
START_CODE_GENERATION("StartCodeGeneration"),
44+
END_CODE_GENERATION("EndCodeGeneration"),
45+
;
46+
47+
override fun toString(): String = operationName
48+
}
49+
50+
enum class MetricDataResult(private val resultName: String) {
51+
SUCCESS("Success"),
52+
FAULT("Fault"),
53+
ERROR("Error"),
54+
LLMFAILURE("LLMFailure"),
55+
;
56+
57+
override fun toString(): String = resultName
58+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
1313
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
1414
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
1515
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
16+
import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
1617
import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
1718
import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
1819
import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
@@ -89,6 +90,33 @@ class FeatureDevClient(
8990
requestBuilder.userContext(featureDevUserContext)
9091
}
9192

93+
fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse =
94+
bearerClient().sendTelemetryEvent { requestBuilder ->
95+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
96+
telemetryEventBuilder.metricData {
97+
it
98+
.metricName("Operation")
99+
.metricValue(1.0)
100+
.timestamp(Instant.now())
101+
.product("Amazon Q For JetBrains")
102+
.dimensions(
103+
listOf(
104+
Dimension.builder()
105+
.name("operationName")
106+
.value(operationName)
107+
.build(),
108+
Dimension.builder()
109+
.name("result")
110+
.value(result)
111+
.build()
112+
)
113+
)
114+
}
115+
}
116+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
117+
requestBuilder.userContext(featureDevUserContext)
118+
}
119+
92120
fun sendFeatureDevCodeGenerationEvent(
93121
conversationId: String,
94122
linesOfCodeGenerated: Int,

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller
66
import com.intellij.notification.NotificationAction
77
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
88
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT
9+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
10+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevException
11+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
12+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
13+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
14+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
15+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
16+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
917
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
1018
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
1119
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType
@@ -63,6 +71,11 @@ suspend fun FeatureDevController.onCodeGeneration(
6371

6472
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code"))
6573

74+
session.sendMetricDataTelemetry(
75+
MetricDataOperationName.START_CODE_GENERATION.toString(),
76+
MetricDataResult.SUCCESS.toString()
77+
)
78+
6679
session.send(message) // Trigger code generation
6780

6881
state = session.sessionState
@@ -135,7 +148,57 @@ suspend fun FeatureDevController.onCodeGeneration(
135148

136149
messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL))
137150

151+
session.sendMetricDataTelemetry(
152+
MetricDataOperationName.END_CODE_GENERATION.toString(),
153+
MetricDataResult.SUCCESS.toString()
154+
)
138155
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
156+
} catch (err: FeatureDevException) {
157+
when (true) {
158+
(err is GuardrailsException) -> {
159+
session.sendMetricDataTelemetry(
160+
MetricDataOperationName.END_CODE_GENERATION.toString(),
161+
MetricDataResult.ERROR.toString()
162+
)
163+
}
164+
(err is PromptRefusalException) -> {
165+
session.sendMetricDataTelemetry(
166+
MetricDataOperationName.END_CODE_GENERATION.toString(),
167+
MetricDataResult.ERROR.toString()
168+
)
169+
}
170+
(err is EmptyPatchException) -> {
171+
session.sendMetricDataTelemetry(
172+
MetricDataOperationName.END_CODE_GENERATION.toString(),
173+
MetricDataResult.LLMFAILURE.toString()
174+
)
175+
}
176+
(err is NoChangeRequiredException) -> {
177+
session.sendMetricDataTelemetry(
178+
MetricDataOperationName.END_CODE_GENERATION.toString(),
179+
MetricDataResult.ERROR.toString()
180+
)
181+
}
182+
(err is ThrottlingException) -> {
183+
session.sendMetricDataTelemetry(
184+
MetricDataOperationName.END_CODE_GENERATION.toString(),
185+
MetricDataResult.ERROR.toString()
186+
)
187+
}
188+
else -> {
189+
session.sendMetricDataTelemetry(
190+
MetricDataOperationName.END_CODE_GENERATION.toString(),
191+
MetricDataResult.FAULT.toString()
192+
)
193+
}
194+
}
195+
throw err
196+
} catch (err: Exception) {
197+
session.sendMetricDataTelemetry(
198+
MetricDataOperationName.END_CODE_GENERATION.toString(),
199+
MetricDataResult.FAULT.toString()
200+
)
201+
throw err
139202
} finally {
140203
if (session.sessionState.token
141204
?.token

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,10 @@ class Session(val tabID: String, val project: Project) {
219219
this._codeResultMessageId = null
220220
}
221221

222+
fun sendMetricDataTelemetry(operationName: String, result: String) {
223+
featureDevService.sendFeatureDevMetricData(operationName, result)
224+
}
225+
222226
suspend fun send(msg: String): Interaction {
223227
// When the task/"thing to do" hasn't been set yet, we want it to be the incoming message
224228
if (task.isEmpty() && msg.isNotEmpty()) {

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project)
232232
}
233233
}
234234

235+
fun sendFeatureDevMetricData(operationName: String, result: String) {
236+
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237+
try {
238+
sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result)
239+
val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId()
240+
logger.debug {
241+
"$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId"
242+
}
243+
} catch (e: Exception) {
244+
logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" }
245+
}
246+
}
247+
235248
fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) {
236249
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237250
try {

‎plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import org.junit.After
2222
import org.junit.Before
2323
import org.junit.Rule
2424
import org.junit.Test
25+
import org.junit.jupiter.api.assertThrows
2526
import org.mockito.kotlin.any
2627
import org.mockito.kotlin.doNothing
2728
import org.mockito.kotlin.doReturn
29+
import org.mockito.kotlin.inOrder
2830
import org.mockito.kotlin.mock
2931
import org.mockito.kotlin.reset
3032
import org.mockito.kotlin.spy
@@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
3638
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
3739
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates
3840
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
41+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
3942
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
43+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
44+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
45+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
46+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
47+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
48+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
4049
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
4150
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
4251
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
@@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC
5059
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
5160
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder
5261
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent
62+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState
5363
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
5464
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed
5565
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction
@@ -423,6 +433,75 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
423433
}
424434
}
425435

436+
@Test
437+
fun `test handleChat onCodeGeneration sends correct metrics for different errors`() = runTest {
438+
data class ErrorTestCase(
439+
val error: Exception,
440+
val expectedMetricResult: MetricDataResult,
441+
)
442+
443+
val testCases = listOf(
444+
ErrorTestCase(
445+
EmptyPatchException("EmptyPatchException", "Empty patch"),
446+
MetricDataResult.LLMFAILURE
447+
),
448+
ErrorTestCase(
449+
GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"),
450+
MetricDataResult.ERROR
451+
),
452+
ErrorTestCase(
453+
PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"),
454+
MetricDataResult.ERROR
455+
),
456+
ErrorTestCase(
457+
NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"),
458+
MetricDataResult.ERROR
459+
),
460+
ErrorTestCase(
461+
ThrottlingException(operation = "GenerateCode", desc = "Request throttled"),
462+
MetricDataResult.ERROR
463+
),
464+
ErrorTestCase(
465+
RuntimeException("Unknown error"),
466+
MetricDataResult.FAULT
467+
)
468+
)
469+
470+
testCases.forEach { (error, expectedResult) ->
471+
val mockSession = mock<Session>()
472+
whenever(mockSession.send(userMessage)).thenThrow(error)
473+
whenever(mockSession.sessionState).thenReturn(
474+
CodeGenerationState(
475+
testTabId,
476+
"",
477+
mock(),
478+
testUploadId,
479+
0,
480+
0.0,
481+
messenger,
482+
token = CancellationTokenSource(),
483+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet())
484+
)
485+
)
486+
487+
assertThrows<Exception> {
488+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
489+
}
490+
491+
val mockInOrder = inOrder(mockSession)
492+
493+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
494+
MetricDataOperationName.START_CODE_GENERATION.toString(),
495+
MetricDataResult.SUCCESS.toString()
496+
497+
)
498+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
499+
MetricDataOperationName.END_CODE_GENERATION.toString(),
500+
expectedResult.toString()
501+
)
502+
}
503+
}
504+
426505
@Test
427506
fun `test processFileClicked handles file rejection`() =
428507
runTest {

0 commit comments

Comments
 (0)