Skip to content

Commit ef104b3

Browse files
siakmun-awskaranA-aws
siakmun-aws
authored and
karanA-aws
committed
telemetry(amazonq): sending metric data in onCodeGeneration (aws#5205)
* telemetry(amazonq): sending metric data in onCodeGeneration
1 parent 0b69fcc commit ef104b3

File tree

6 files changed

+222
-1
lines changed

6 files changed

+222
-1
lines changed

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,21 @@ enum class FeatureDevOperation(private val operationName: String) {
4141

4242
override fun toString(): String = operationName
4343
}
44+
45+
enum class MetricDataOperationName(private val operationName: String) {
46+
StartCodeGeneration("StartCodeGeneration"),
47+
EndCodeGeneration("EndCodeGeneration"),
48+
;
49+
50+
override fun toString(): String = operationName
51+
}
52+
53+
enum class MetricDataResult(private val resultName: String) {
54+
Success("Success"),
55+
Fault("Fault"),
56+
Error("Error"),
57+
LlmFailure("LLMFailure"),
58+
;
59+
60+
override fun toString(): String = resultName
61+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
1313
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
1414
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
1515
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
16+
import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
1617
import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
1718
import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
1819
import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
@@ -89,6 +90,33 @@ class FeatureDevClient(
8990
requestBuilder.userContext(featureDevUserContext)
9091
}
9192

93+
fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse =
94+
bearerClient().sendTelemetryEvent { requestBuilder ->
95+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
96+
telemetryEventBuilder.metricData {
97+
it
98+
.metricName("Operation")
99+
.metricValue(1.0)
100+
.timestamp(Instant.now())
101+
.product("FeatureDev")
102+
.dimensions(
103+
listOf(
104+
Dimension.builder()
105+
.name("operationName")
106+
.value(operationName)
107+
.build(),
108+
Dimension.builder()
109+
.name("result")
110+
.value(result)
111+
.build()
112+
)
113+
)
114+
}
115+
}
116+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
117+
requestBuilder.userContext(featureDevUserContext)
118+
}
119+
92120
fun sendFeatureDevCodeGenerationEvent(
93121
conversationId: String,
94122
linesOfCodeGenerated: Int,

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,13 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller
66
import com.intellij.notification.NotificationAction
77
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
88
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT
9+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
10+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
11+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
12+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
13+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
14+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
15+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
916
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
1017
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
1118
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType
@@ -63,6 +70,11 @@ suspend fun FeatureDevController.onCodeGeneration(
6370

6471
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code"))
6572

73+
session.sendMetricDataTelemetry(
74+
MetricDataOperationName.StartCodeGeneration,
75+
MetricDataResult.Success
76+
)
77+
6678
session.send(message) // Trigger code generation
6779

6880
state = session.sessionState
@@ -134,8 +146,29 @@ suspend fun FeatureDevController.onCodeGeneration(
134146
}
135147

136148
messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL))
137-
138149
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
150+
} catch (err: Exception) {
151+
when (err) {
152+
is GuardrailsException, is NoChangeRequiredException, is PromptRefusalException, is ThrottlingException -> {
153+
session.sendMetricDataTelemetry(
154+
MetricDataOperationName.EndCodeGeneration,
155+
MetricDataResult.Error
156+
)
157+
}
158+
is EmptyPatchException -> {
159+
session.sendMetricDataTelemetry(
160+
MetricDataOperationName.EndCodeGeneration,
161+
MetricDataResult.LlmFailure
162+
)
163+
}
164+
else -> {
165+
session.sendMetricDataTelemetry(
166+
MetricDataOperationName.EndCodeGeneration,
167+
MetricDataResult.Fault
168+
)
169+
}
170+
}
171+
throw err
139172
} finally {
140173
if (session.sessionState.token
141174
?.token
@@ -155,6 +188,11 @@ suspend fun FeatureDevController.onCodeGeneration(
155188
)
156189
}
157190
}
191+
192+
session.sendMetricDataTelemetry(
193+
MetricDataOperationName.EndCodeGeneration,
194+
MetricDataResult.Success
195+
)
158196
}
159197

160198
private suspend fun disposeToken(

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATIO
1414
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ConversationIdNotFoundException
1515
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_NAME
1616
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MAX_PROJECT_SIZE_BYTES
17+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
18+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
1719
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
1820
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.IncomingFeatureDevMessage
1921
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendAsyncEventProgress
@@ -216,6 +218,10 @@ class Session(val tabID: String, val project: Project) {
216218
this._codeResultMessageId = null
217219
}
218220

221+
fun sendMetricDataTelemetry(operationName: MetricDataOperationName, result: MetricDataResult) {
222+
featureDevService.sendFeatureDevMetricData(operationName.toString(), result.toString())
223+
}
224+
219225
suspend fun send(msg: String): Interaction {
220226
// When the task/"thing to do" hasn't been set yet, we want it to be the incoming message
221227
if (task.isEmpty() && msg.isNotEmpty()) {

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project)
232232
}
233233
}
234234

235+
fun sendFeatureDevMetricData(operationName: String, result: String) {
236+
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237+
try {
238+
sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result)
239+
val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId()
240+
logger.debug {
241+
"$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId"
242+
}
243+
} catch (e: Exception) {
244+
logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" }
245+
}
246+
}
247+
235248
fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) {
236249
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237250
try {

‎plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import org.junit.After
2222
import org.junit.Before
2323
import org.junit.Rule
2424
import org.junit.Test
25+
import org.junit.jupiter.api.assertThrows
2526
import org.mockito.kotlin.any
2627
import org.mockito.kotlin.doNothing
2728
import org.mockito.kotlin.doReturn
29+
import org.mockito.kotlin.inOrder
2830
import org.mockito.kotlin.mock
2931
import org.mockito.kotlin.reset
3032
import org.mockito.kotlin.spy
@@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
3638
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
3739
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates
3840
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
41+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
3942
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
43+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
44+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
45+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
46+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
47+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
48+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
4049
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
4150
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
4251
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
@@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC
5059
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
5160
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder
5261
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent
62+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState
5363
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
5464
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed
5565
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction
@@ -424,6 +434,114 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
424434
}
425435
}
426436

437+
@Test
438+
fun `test handleChat onCodeGeneration sends success metrics`() = runTest {
439+
val mockSession = mock<Session>()
440+
val featureDevService = mockk<FeatureDevService>()
441+
val repoContext = mock<FeatureDevSessionContext>()
442+
val sessionStateConfig = SessionStateConfig(testConversationId, repoContext, featureDevService)
443+
val mockInteraction = mock<Interaction>()
444+
whenever(mockSession.send(userMessage)).thenReturn(mockInteraction)
445+
whenever(mockSession.sessionState).thenReturn(
446+
PrepareCodeGenerationState(
447+
testTabId,
448+
CancellationTokenSource(),
449+
"test-command",
450+
sessionStateConfig,
451+
newFileContents,
452+
deletedFiles,
453+
testReferences,
454+
testUploadId,
455+
0,
456+
messenger,
457+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()),
458+
),
459+
)
460+
461+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
462+
463+
val mockInOrder = inOrder(mockSession)
464+
465+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
466+
MetricDataOperationName.StartCodeGeneration,
467+
MetricDataResult.Success
468+
469+
)
470+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
471+
MetricDataOperationName.EndCodeGeneration,
472+
MetricDataResult.Success
473+
)
474+
}
475+
476+
@Test
477+
fun `test handleChat onCodeGeneration sends correct failure metrics for different errors`() = runTest {
478+
data class ErrorTestCase(
479+
val error: Exception,
480+
val expectedMetricResult: MetricDataResult,
481+
)
482+
483+
val testCases = listOf(
484+
ErrorTestCase(
485+
EmptyPatchException("EmptyPatchException", "Empty patch"),
486+
MetricDataResult.LlmFailure
487+
),
488+
ErrorTestCase(
489+
GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"),
490+
MetricDataResult.Error
491+
),
492+
ErrorTestCase(
493+
PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"),
494+
MetricDataResult.Error
495+
),
496+
ErrorTestCase(
497+
NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"),
498+
MetricDataResult.Error
499+
),
500+
ErrorTestCase(
501+
ThrottlingException(operation = "GenerateCode", desc = "Request throttled"),
502+
MetricDataResult.Error
503+
),
504+
ErrorTestCase(
505+
RuntimeException("Unknown error"),
506+
MetricDataResult.Fault
507+
)
508+
)
509+
510+
testCases.forEach { (error, expectedResult) ->
511+
val mockSession = mock<Session>()
512+
whenever(mockSession.send(userMessage)).thenThrow(error)
513+
whenever(mockSession.sessionState).thenReturn(
514+
CodeGenerationState(
515+
testTabId,
516+
"",
517+
mock(),
518+
testUploadId,
519+
0,
520+
0.0,
521+
messenger,
522+
token = CancellationTokenSource(),
523+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet())
524+
)
525+
)
526+
527+
assertThrows<Exception> {
528+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
529+
}
530+
531+
val mockInOrder = inOrder(mockSession)
532+
533+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
534+
MetricDataOperationName.StartCodeGeneration,
535+
MetricDataResult.Success
536+
537+
)
538+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
539+
MetricDataOperationName.EndCodeGeneration,
540+
expectedResult
541+
)
542+
}
543+
}
544+
427545
@Test
428546
fun `test processFileClicked handles file rejection`() =
429547
runTest {

0 commit comments

Comments
 (0)