Skip to content

Commit 392f487

Browse files
ctlai95ctlai95
authored
Merge branch 'main' into fix-telemetry
2 parents 4b78b0a + 5abfb19 commit 392f487

File tree

9 files changed

+230
-2
lines changed

9 files changed

+230
-2
lines changed

‎.changes/next-release/bugfix-58a31adb-0d87-4a2d-ac83-9b5846e023e6.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"type" : "bugfix",
3+
"description" : "/transform: use correct doc link in SQL conversion help message"
4+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,21 @@ enum class FeatureDevOperation(private val operationName: String) {
3838

3939
override fun toString(): String = operationName
4040
}
41+
42+
enum class MetricDataOperationName(private val operationName: String) {
43+
StartCodeGeneration("StartCodeGeneration"),
44+
EndCodeGeneration("EndCodeGeneration"),
45+
;
46+
47+
override fun toString(): String = operationName
48+
}
49+
50+
enum class MetricDataResult(private val resultName: String) {
51+
Success("Success"),
52+
Fault("Fault"),
53+
Error("Error"),
54+
LlmFailure("LLMFailure"),
55+
;
56+
57+
override fun toString(): String = resultName
58+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
1313
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
1414
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
1515
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
16+
import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
1617
import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
1718
import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
1819
import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
@@ -89,6 +90,33 @@ class FeatureDevClient(
8990
requestBuilder.userContext(featureDevUserContext)
9091
}
9192

93+
fun sendFeatureDevMetricData(operationName: String, result: String): SendTelemetryEventResponse =
94+
bearerClient().sendTelemetryEvent { requestBuilder ->
95+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
96+
telemetryEventBuilder.metricData {
97+
it
98+
.metricName("Operation")
99+
.metricValue(1.0)
100+
.timestamp(Instant.now())
101+
.product("FeatureDev")
102+
.dimensions(
103+
listOf(
104+
Dimension.builder()
105+
.name("operationName")
106+
.value(operationName)
107+
.build(),
108+
Dimension.builder()
109+
.name("result")
110+
.value(result)
111+
.build()
112+
)
113+
)
114+
}
115+
}
116+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
117+
requestBuilder.userContext(featureDevUserContext)
118+
}
119+
92120
fun sendFeatureDevCodeGenerationEvent(
93121
conversationId: String,
94122
linesOfCodeGenerated: Int,

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerExtensions.kt‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,13 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.controller
66
import com.intellij.notification.NotificationAction
77
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
88
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATION_RETRY_LIMIT
9+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
10+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
11+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
12+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
13+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
14+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
15+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
916
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
1017
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
1118
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUpStatusType
@@ -63,6 +70,11 @@ suspend fun FeatureDevController.onCodeGeneration(
6370

6471
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.generating_code"))
6572

73+
session.sendMetricDataTelemetry(
74+
MetricDataOperationName.StartCodeGeneration,
75+
MetricDataResult.Success
76+
)
77+
6678
session.send(message) // Trigger code generation
6779

6880
state = session.sessionState
@@ -134,8 +146,29 @@ suspend fun FeatureDevController.onCodeGeneration(
134146
}
135147

136148
messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase, InsertAction.ALL))
137-
138149
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
150+
} catch (err: Exception) {
151+
when (err) {
152+
is GuardrailsException, is NoChangeRequiredException, is PromptRefusalException, is ThrottlingException -> {
153+
session.sendMetricDataTelemetry(
154+
MetricDataOperationName.EndCodeGeneration,
155+
MetricDataResult.Error
156+
)
157+
}
158+
is EmptyPatchException -> {
159+
session.sendMetricDataTelemetry(
160+
MetricDataOperationName.EndCodeGeneration,
161+
MetricDataResult.LlmFailure
162+
)
163+
}
164+
else -> {
165+
session.sendMetricDataTelemetry(
166+
MetricDataOperationName.EndCodeGeneration,
167+
MetricDataResult.Fault
168+
)
169+
}
170+
}
171+
throw err
139172
} finally {
140173
if (session.sessionState.token
141174
?.token
@@ -155,6 +188,11 @@ suspend fun FeatureDevController.onCodeGeneration(
155188
)
156189
}
157190
}
191+
192+
session.sendMetricDataTelemetry(
193+
MetricDataOperationName.EndCodeGeneration,
194+
MetricDataResult.Success
195+
)
158196
}
159197

160198
private suspend fun disposeToken(

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CODE_GENERATIO
1414
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ConversationIdNotFoundException
1515
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_NAME
1616
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MAX_PROJECT_SIZE_BYTES
17+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
18+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
1719
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
1820
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.IncomingFeatureDevMessage
1921
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendAsyncEventProgress
@@ -219,6 +221,10 @@ class Session(val tabID: String, val project: Project) {
219221
this._codeResultMessageId = null
220222
}
221223

224+
fun sendMetricDataTelemetry(operationName: MetricDataOperationName, result: MetricDataResult) {
225+
featureDevService.sendFeatureDevMetricData(operationName.toString(), result.toString())
226+
}
227+
222228
suspend fun send(msg: String): Interaction {
223229
// When the task/"thing to do" hasn't been set yet, we want it to be the incoming message
224230
if (task.isEmpty() && msg.isNotEmpty()) {

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,19 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project)
232232
}
233233
}
234234

235+
fun sendFeatureDevMetricData(operationName: String, result: String) {
236+
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237+
try {
238+
sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevMetricData(operationName, result)
239+
val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId()
240+
logger.debug {
241+
"$FEATURE_NAME: succesfully sent feature dev metric data: OperationName: $operationName Result: $result RequestId: $requestId"
242+
}
243+
} catch (e: Exception) {
244+
logger.warn(e) { "$FEATURE_NAME: failed to send feature dev metric data" }
245+
}
246+
}
247+
235248
fun sendFeatureDevCodeGenerationEvent(conversationId: String, linesOfCodeGenerated: Int, charactersOfCodeGenerated: Int) {
236249
val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
237250
try {

‎plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import org.junit.After
2222
import org.junit.Before
2323
import org.junit.Rule
2424
import org.junit.Test
25+
import org.junit.jupiter.api.assertThrows
2526
import org.mockito.kotlin.any
2627
import org.mockito.kotlin.doNothing
2728
import org.mockito.kotlin.doReturn
29+
import org.mockito.kotlin.inOrder
2830
import org.mockito.kotlin.mock
2931
import org.mockito.kotlin.reset
3032
import org.mockito.kotlin.spy
@@ -36,7 +38,14 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
3638
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
3739
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededStates
3840
import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher
41+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.EmptyPatchException
3942
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
43+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.GuardrailsException
44+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataOperationName
45+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MetricDataResult
46+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.NoChangeRequiredException
47+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.PromptRefusalException
48+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ThrottlingException
4049
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients.FeatureDevClient
4150
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FeatureDevMessageType
4251
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.FollowUp
@@ -50,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendC
5059
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
5160
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendUpdatePlaceholder
5261
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.updateFileComponent
62+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeGenerationState
5363
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
5464
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DiffMetricsProcessed
5565
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Interaction
@@ -423,6 +433,114 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
423433
}
424434
}
425435

436+
@Test
437+
fun `test handleChat onCodeGeneration sends success metrics`() = runTest {
438+
val mockSession = mock<Session>()
439+
val featureDevService = mockk<FeatureDevService>()
440+
val repoContext = mock<FeatureDevSessionContext>()
441+
val sessionStateConfig = SessionStateConfig(testConversationId, repoContext, featureDevService)
442+
val mockInteraction = mock<Interaction>()
443+
whenever(mockSession.send(userMessage)).thenReturn(mockInteraction)
444+
whenever(mockSession.sessionState).thenReturn(
445+
PrepareCodeGenerationState(
446+
testTabId,
447+
CancellationTokenSource(),
448+
"test-command",
449+
sessionStateConfig,
450+
newFileContents,
451+
deletedFiles,
452+
testReferences,
453+
testUploadId,
454+
0,
455+
messenger,
456+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet()),
457+
),
458+
)
459+
460+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
461+
462+
val mockInOrder = inOrder(mockSession)
463+
464+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
465+
MetricDataOperationName.StartCodeGeneration,
466+
MetricDataResult.Success
467+
468+
)
469+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
470+
MetricDataOperationName.EndCodeGeneration,
471+
MetricDataResult.Success
472+
)
473+
}
474+
475+
@Test
476+
fun `test handleChat onCodeGeneration sends correct failure metrics for different errors`() = runTest {
477+
data class ErrorTestCase(
478+
val error: Exception,
479+
val expectedMetricResult: MetricDataResult,
480+
)
481+
482+
val testCases = listOf(
483+
ErrorTestCase(
484+
EmptyPatchException("EmptyPatchException", "Empty patch"),
485+
MetricDataResult.LlmFailure
486+
),
487+
ErrorTestCase(
488+
GuardrailsException(operation = "GenerateCode", desc = "Failed guardrails"),
489+
MetricDataResult.Error
490+
),
491+
ErrorTestCase(
492+
PromptRefusalException(operation = "GenerateCode", desc = "Prompt refused"),
493+
MetricDataResult.Error
494+
),
495+
ErrorTestCase(
496+
NoChangeRequiredException(operation = "GenerateCode", desc = "No changes needed"),
497+
MetricDataResult.Error
498+
),
499+
ErrorTestCase(
500+
ThrottlingException(operation = "GenerateCode", desc = "Request throttled"),
501+
MetricDataResult.Error
502+
),
503+
ErrorTestCase(
504+
RuntimeException("Unknown error"),
505+
MetricDataResult.Fault
506+
)
507+
)
508+
509+
testCases.forEach { (error, expectedResult) ->
510+
val mockSession = mock<Session>()
511+
whenever(mockSession.send(userMessage)).thenThrow(error)
512+
whenever(mockSession.sessionState).thenReturn(
513+
CodeGenerationState(
514+
testTabId,
515+
"",
516+
mock(),
517+
testUploadId,
518+
0,
519+
0.0,
520+
messenger,
521+
token = CancellationTokenSource(),
522+
diffMetricsProcessed = DiffMetricsProcessed(HashSet(), HashSet())
523+
)
524+
)
525+
526+
assertThrows<Exception> {
527+
controller.onCodeGeneration(mockSession, userMessage, testTabId)
528+
}
529+
530+
val mockInOrder = inOrder(mockSession)
531+
532+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
533+
MetricDataOperationName.StartCodeGeneration,
534+
MetricDataResult.Success
535+
536+
)
537+
mockInOrder.verify(mockSession).sendMetricDataTelemetry(
538+
MetricDataOperationName.EndCodeGeneration,
539+
expectedResult
540+
)
541+
}
542+
}
543+
426544
@Test
427545
fun `test processFileClicked handles file rejection`() =
428546
runTest {

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/controller/CodeTransformChatController.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
2525
import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthFollowUpType
2626
import software.aws.toolkits.jetbrains.services.codemodernizer.ArtifactHandler
2727
import software.aws.toolkits.jetbrains.services.codemodernizer.CodeModernizerManager
28+
import software.aws.toolkits.jetbrains.services.codemodernizer.CodeModernizerManager.Companion.LOG
2829
import software.aws.toolkits.jetbrains.services.codemodernizer.CodeTransformTelemetryManager
2930
import software.aws.toolkits.jetbrains.services.codemodernizer.EXPLAINABILITY_V1
3031
import software.aws.toolkits.jetbrains.services.codemodernizer.HilTelemetryMetaData
@@ -680,6 +681,7 @@ class CodeTransformChatController(
680681
)
681682

682683
private suspend fun handleCodeTransformResult(result: CodeModernizerJobCompletedResult) {
684+
LOG.info { "CodeModernizerJobCompletedResult: $result" }
683685
when (result) {
684686
is CodeModernizerJobCompletedResult.Stopped, CodeModernizerJobCompletedResult.JobAbortedBeforeStarting -> handleCodeTransformStoppedByUser()
685687
is CodeModernizerJobCompletedResult.JobFailed -> handleCodeTransformJobFailed(result.failureReason)
@@ -692,6 +694,7 @@ class CodeTransformChatController(
692694
CodeModernizerSessionState.getInstance(context.project).currentJobId as JobId,
693695
TransformationDownloadArtifactType.CLIENT_INSTRUCTIONS
694696
)
697+
LOG.info { "Download result: $downloadResult" }
695698
when (downloadResult) {
696699
is DownloadArtifactResult.Success -> {
697700
if (downloadResult.artifact !is CodeModernizerArtifact) return artifactHandler.notifyUnableToApplyPatch("")

‎plugins/core/resources/resources/software/aws/toolkits/resources/MessagesBundle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -602,7 +602,7 @@ codemodernizer.builderrordialog.description.title=Error occurred when building y
602602
codemodernizer.chat.form.user_selection.item.choose_module=Choose a module to transform
603603
codemodernizer.chat.form.user_selection.item.choose_one_or_multiple_diffs_option=Choose how to receive proposed changes
604604
codemodernizer.chat.form.user_selection.item.choose_skip_tests_option=Choose to skip unit tests
605-
codemodernizer.chat.form.user_selection.item.choose_sql_metadata_file=Okay, I can convert the embedded SQL code for your Oracle to PostgreSQL transformation. To get started, upload the zipped metadata file from your schema conversion in AWS Data Migration Service (DMS). To retrieve the metadata file:\n1. Open your database migration project in the AWS DMS console.\n2. Open the schema conversion and choose **Convert the embedded SQL in your application**.\n3. Once you complete the conversion, close the project and go to the S3 bucket where your project is stored.\n4. Open the folder and find the project folder ("sct-project").\n5. Download the object inside the project folder. This will be a zip file.\n\nFor more info, refer to the [documentation](https://docs.aws.amazon.com/dms/latest/userguide/schema-conversion-save-apply.html#schema-conversion-save).
605+
codemodernizer.chat.form.user_selection.item.choose_sql_metadata_file=Okay, I can convert the embedded SQL code for your Oracle to PostgreSQL transformation. To get started, upload the zipped metadata file from your schema conversion in AWS Data Migration Service (DMS). To retrieve the metadata file:\n1. Open your database migration project in the AWS DMS console.\n2. Open the schema conversion and choose **Convert the embedded SQL in your application**.\n3. Once you complete the conversion, close the project and go to the S3 bucket where your project is stored.\n4. Open the folder and find the project folder ("sct-project").\n5. Download the object inside the project folder. This will be a zip file.\n\nFor more info, refer to the [documentation](https://docs.aws.amazon.com/dms/latest/userguide/schema-conversion-embedded-sql.html).
606606
codemodernizer.chat.form.user_selection.item.choose_target_version=Choose the target code version
607607
codemodernizer.chat.form.user_selection.title=Q - Code transformation
608608
codemodernizer.chat.message.absolute_path_detected=I detected {0} potential absolute file path(s) in your {1} file: **{2}**. Absolute file paths might cause issues when I build your code. Any errors will show up in the build log.

0 commit comments

Comments
 (0)