Skip to content

Commit 3e44279

Browse files
siakmun-awssiakmun-aws
authored
telemetry(amazonq): sending metric data in onCodeGeneration (#5205)
* telemetry(amazonq): sending metric data in onCodeGeneration
1 parent 297cb06 commit 3e44279

File tree

6 files changed

+222
-1
lines changed

6 files changed

+222
-1
lines changed

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,21 @@ enum class FeatureDevOperation(private val operationName: String) {
3838

3939
override fun toString(): String = operationName
4040
}
41+
42+
enum class MetricDataOperationName(private val operationName: String) {
43+
StartCodeGeneration("StartCodeGeneration"),
44+
EndCodeGeneration("EndCodeGeneration"),
45+
;
46+
47+
override fun toString(): String = operationName
48+
}
49+
50+
enum class MetricDataResult(private val resultName: String) {
51+
Success("Success"),
52+
Fault("Fault"),
53+
Error("Error"),
54+
LlmFailure("LLMFailure"),
55+
;
56+
57+
override fun toString(): String = resultName
58+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
1313
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
1414
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
1515
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
16+
import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
1617
import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
1718
import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
1819
import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
@@ -89,6 +90,33 @@ class FeatureDevClient(
8990
requestBuilder.userContext(featureDevUserContext)
9091
}
9192

93+
fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse =
94+
bearerClient().sendTelemetryEvent { requestBuilder ->
95+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
96+
telemetryEventBuilder.metricData {
97+
it
98+
.metricName("Operation")
99+
.metricValue(1.0)
100+
.timestamp(Instant.now())
101+
.product("FeatureDev")
102+
.dimensions(
103+
listOf(
104+
Dimension.builder()
105+
.name("operationName")
106+
.value(operationName)
107+
.build(),
108+
Dimension.builder()
109+
.name("result")
110+
.value(result)
111+
.build()
112+
)
113+
)
114+
}
115+
}
116+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
117+
requestBuilder.userContext(featureDevUserContext)
118+
}
119+
92120
fun sendFeatureDevCodeGenerationEvent(
93121
conversationId: String,
94122
linesOfCodeGenerated: Int,

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,13 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller
66
import com.intellij.notification.NotificationAction
77
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
88
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT
9+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
10+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
11+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
12+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
13+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
14+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
15+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
916
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
1017
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
1118
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType
@@ -63,6 +70,11 @@ suspend fun FeatureDevController.onCodeGeneration(
6370

6471
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code"))
6572

73+
session.sendMetricDataTelemetry(
74+
MetricDataOperationName.StartCodeGeneration,
75+
MetricDataResult.Success
76+
)
77+
6678
session.send(message) // Trigger code generation
6779

6880
state = session.sessionState
@@ -134,8 +146,29 @@ suspend fun FeatureDevController.onCodeGeneration(
134146
}
135147

136148
messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL))
137-
138149
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
150+
} catch (err: Exception) {
151+
when (err) {
152+
is GuardrailsException, is NoChangeRequiredException, is PromptRefusalException, is ThrottlingException -> {
153+
session.sendMetricDataTelemetry(
154+
MetricDataOperationName.EndCodeGeneration,
155+
MetricDataResult.Error
156+
)
157+
}
158+
is EmptyPatchException -> {
159+
session.sendMetricDataTelemetry(
160+
MetricDataOperationName.EndCodeGeneration,
161+
MetricDataResult.LlmFailure
162+
)
163+
}
164+
else -> {
165+
session.sendMetricDataTelemetry(
166+
MetricDataOperationName.EndCodeGeneration,
167+
MetricDataResult.Fault
168+
)
169+
}
170+
}
171+
throw err
139172
} finally {
140173
if (session.sessionState.token
141174
?.token
@@ -155,6 +188,11 @@ suspend fun FeatureDevController.onCodeGeneration(
155188
)
156189
}
157190
}
191+
192+
session.sendMetricDataTelemetry(
193+
MetricDataOperationName.EndCodeGeneration,
194+
MetricDataResult.Success
195+
)
158196
}
159197

160198
private suspend fun disposeToken(

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATIO
1414
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ConversationIdNotFoundException
1515
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_NAME
1616
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MAX_PROJECT_SIZE_BYTES
17+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
18+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
1719
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
1820
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.IncomingFeatureDevMessage
1921
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendAsyncEventProgress
@@ -219,6 +221,10 @@ class Session(val tabID: String, val project: Project) {
219221
this._codeResultMessageId = null
220222
}
221223

224+
fun sendMetricDataTelemetry(operationName: MetricDataOperationName, result: MetricDataResult) {
225+
featureDevService.sendFeatureDevMetricData(operationName.toString(), result.toString())
226+
}
227+
222228
suspend fun send(msg: String): Interaction {
223229
// When the task/"thing to do" hasn't been set yet, we want it to be the incoming message
224230
if (task.isEmpty() && msg.isNotEmpty()) {

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project)
232232
}
233233
}
234234

235+
fun sendFeatureDevMetricData(operationName: String, result: String) {
236+
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237+
try {
238+
sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result)
239+
val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId()
240+
logger.debug {
241+
"$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId"
242+
}
243+
} catch (e: Exception) {
244+
logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" }
245+
}
246+
}
247+
235248
fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) {
236249
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237250
try {

‎plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import org.junit.After
2222
import org.junit.Before
2323
import org.junit.Rule
2424
import org.junit.Test
25+
import org.junit.jupiter.api.assertThrows
2526
import org.mockito.kotlin.any
2627
import org.mockito.kotlin.doNothing
2728
import org.mockito.kotlin.doReturn
29+
import org.mockito.kotlin.inOrder
2830
import org.mockito.kotlin.mock
2931
import org.mockito.kotlin.reset
3032
import org.mockito.kotlin.spy
@@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
3638
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
3739
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates
3840
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
41+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
3942
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
43+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
44+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
45+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
46+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
47+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
48+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
4049
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
4150
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
4251
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
@@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC
5059
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
5160
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder
5261
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent
62+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState
5363
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
5464
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed
5565
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction
@@ -423,6 +433,114 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
423433
}
424434
}
425435

436+
@Test
437+
fun `test handleChat onCodeGeneration sends success metrics`() = runTest {
438+
val mockSession = mock<Session>()
439+
val featureDevService = mockk<FeatureDevService>()
440+
val repoContext = mock<FeatureDevSessionContext>()
441+
val sessionStateConfig = SessionStateConfig(testConversationId, repoContext, featureDevService)
442+
val mockInteraction = mock<Interaction>()
443+
whenever(mockSession.send(userMessage)).thenReturn(mockInteraction)
444+
whenever(mockSession.sessionState).thenReturn(
445+
PrepareCodeGenerationState(
446+
testTabId,
447+
CancellationTokenSource(),
448+
"test-command",
449+
sessionStateConfig,
450+
newFileContents,
451+
deletedFiles,
452+
testReferences,
453+
testUploadId,
454+
0,
455+
messenger,
456+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()),
457+
),
458+
)
459+
460+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
461+
462+
val mockInOrder = inOrder(mockSession)
463+
464+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
465+
MetricDataOperationName.StartCodeGeneration,
466+
MetricDataResult.Success
467+
468+
)
469+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
470+
MetricDataOperationName.EndCodeGeneration,
471+
MetricDataResult.Success
472+
)
473+
}
474+
475+
@Test
476+
fun `test handleChat onCodeGeneration sends correct failure metrics for different errors`() = runTest {
477+
data class ErrorTestCase(
478+
val error: Exception,
479+
val expectedMetricResult: MetricDataResult,
480+
)
481+
482+
val testCases = listOf(
483+
ErrorTestCase(
484+
EmptyPatchException("EmptyPatchException", "Empty patch"),
485+
MetricDataResult.LlmFailure
486+
),
487+
ErrorTestCase(
488+
GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"),
489+
MetricDataResult.Error
490+
),
491+
ErrorTestCase(
492+
PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"),
493+
MetricDataResult.Error
494+
),
495+
ErrorTestCase(
496+
NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"),
497+
MetricDataResult.Error
498+
),
499+
ErrorTestCase(
500+
ThrottlingException(operation = "GenerateCode", desc = "Request throttled"),
501+
MetricDataResult.Error
502+
),
503+
ErrorTestCase(
504+
RuntimeException("Unknown error"),
505+
MetricDataResult.Fault
506+
)
507+
)
508+
509+
testCases.forEach { (error, expectedResult) ->
510+
val mockSession = mock<Session>()
511+
whenever(mockSession.send(userMessage)).thenThrow(error)
512+
whenever(mockSession.sessionState).thenReturn(
513+
CodeGenerationState(
514+
testTabId,
515+
"",
516+
mock(),
517+
testUploadId,
518+
0,
519+
0.0,
520+
messenger,
521+
token = CancellationTokenSource(),
522+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet())
523+
)
524+
)
525+
526+
assertThrows<Exception> {
527+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
528+
}
529+
530+
val mockInOrder = inOrder(mockSession)
531+
532+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
533+
MetricDataOperationName.StartCodeGeneration,
534+
MetricDataResult.Success
535+
536+
)
537+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
538+
MetricDataOperationName.EndCodeGeneration,
539+
expectedResult
540+
)
541+
}
542+
}
543+
426544
@Test
427545
fun `test processFileClicked handles file rejection`() =
428546
runTest {

0 commit comments

Comments
 (0)