Skip to content

Commit 9fc070a

Browse files
KevinDing1KevinDing1
authored
telemetry(amazonq): Added metric data for /doc and improved error classification (#5498)
1 parent a4f5c07 commit 9fc070a

File tree

8 files changed

+286
-47
lines changed

8 files changed

+286
-47
lines changed

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/common/clients/AmazonQCodeGenerateClient.kt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
1313
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
1414
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
1515
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
16+
import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
1617
import software.amazon.awssdk.services.codewhispererruntime.model.DocV2AcceptanceEvent
1718
import software.amazon.awssdk.services.codewhispererruntime.model.DocV2GenerationEvent
1819
import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
@@ -89,6 +90,33 @@ class AmazonQCodeGenerateClient(private val project: Project) {
8990
requestBuilder.userContext(docUserContext)
9091
}
9192

93+
fun sendDocMetricData(operationName: String, result: String): SendTelemetryEventResponse =
94+
bearerClient().sendTelemetryEvent { requestBuilder ->
95+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
96+
telemetryEventBuilder.metricData {
97+
it
98+
.metricName("Operation")
99+
.metricValue(1.0)
100+
.timestamp(Instant.now())
101+
.product("DocGeneration")
102+
.dimensions(
103+
listOf(
104+
Dimension.builder()
105+
.name("operationName")
106+
.value(operationName)
107+
.build(),
108+
Dimension.builder()
109+
.name("result")
110+
.value(result)
111+
.build()
112+
)
113+
)
114+
}
115+
}
116+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
117+
requestBuilder.userContext(docUserContext)
118+
}
119+
92120
fun createTaskAssistConversation(): CreateTaskAssistConversationResponse = bearerClient().createTaskAssistConversation(
93121
CreateTaskAssistConversationRequest.builder().build()
94122
)

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/common/util/AmazonQCodeGenService.kt

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import software.aws.toolkits.core.utils.getLogger
2020
import software.aws.toolkits.core.utils.warn
2121
import software.aws.toolkits.jetbrains.common.clients.AmazonQCodeGenerateClient
2222
import software.aws.toolkits.jetbrains.common.session.Intent
23-
import software.aws.toolkits.jetbrains.services.amazonqDoc.docServiceError
2423
import software.aws.toolkits.jetbrains.services.amazonqDoc.session.DocGenerationStreamResult
2524
import software.aws.toolkits.jetbrains.services.amazonqDoc.session.ExportDocTaskAssistResultArchiveStreamResult
2625
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ApiException
@@ -34,9 +33,9 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MonthlyConvers
3433
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ServiceException
3534
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ZipFileCorruptedException
3635
import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
37-
import software.aws.toolkits.resources.message
3836
import software.aws.toolkits.telemetry.AmazonqTelemetry
3937
import software.aws.toolkits.telemetry.MetricResult
38+
import software.aws.toolkits.jetbrains.services.amazonqDoc.ContentLengthException as DocContentLengthException
4039

4140
class AmazonQCodeGenService(val proxyClient: AmazonQCodeGenerateClient, val project: Project) {
4241
fun createConversation(): String {
@@ -115,7 +114,7 @@ class AmazonQCodeGenService(val proxyClient: AmazonQCodeGenerateClient, val proj
115114

116115
if (e is ValidationException && e.message?.contains("Invalid contentLength") == true) {
117116
if (featureName?.equals("docGeneration") == true) {
118-
throw docServiceError(message("amazonqDoc.exception.content_length_error"))
117+
throw DocContentLengthException(operation = FeatureDevOperation.CreateUploadUrl.toString(), desc = null, cause = e.cause)
119118
}
120119
throw ContentLengthException(operation = FeatureDevOperation.CreateUploadUrl.toString(), desc = null, cause = e.cause)
121120
}
@@ -162,6 +161,9 @@ class AmazonQCodeGenService(val proxyClient: AmazonQCodeGenerateClient, val proj
162161
) {
163162
throw CodeIterationLimitException(operation = FeatureDevOperation.StartTaskAssistCodeGeneration.toString(), desc = null, e.cause)
164163
} else if (e is ValidationException && e.message?.contains("repo size is exceeding the limits") == true) {
164+
if (intent == Intent.DOC) {
165+
throw DocContentLengthException(operation = FeatureDevOperation.CreateUploadUrl.toString(), desc = null, cause = e.cause)
166+
}
165167
throw ContentLengthException(operation = FeatureDevOperation.StartTaskAssistCodeGeneration.toString(), desc = null, cause = e.cause)
166168
} else if (e is ValidationException && e.message?.contains("zipped file is corrupted") == true) {
167169
throw ZipFileCorruptedException(operation = FeatureDevOperation.StartTaskAssistCodeGeneration.toString(), desc = null, e.cause)

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/DocConstants.kt

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,21 @@ const val DIAGRAM_SVG_EXT = "svg"
2121
const val DIAGRAM_DOT_EXT = "dot"
2222
val SUPPORTED_DIAGRAM_EXT_SET: Set<String> = setOf(DIAGRAM_SVG_EXT, DIAGRAM_DOT_EXT)
2323
val SUPPORTED_DIAGRAM_FILE_NAME_SET: Set<String> = SUPPORTED_DIAGRAM_EXT_SET.map { INFRA_DIAGRAM_PREFIX + it }.toSet()
24+
25+
enum class MetricDataOperationName(private val operationName: String) {
26+
StartDocGeneration("StartDocGeneration"),
27+
EndDocGeneration("EndDocGeneration"),
28+
;
29+
30+
override fun toString(): String = operationName
31+
}
32+
33+
enum class MetricDataResult(private val resultName: String) {
34+
Success("Success"),
35+
Fault("Fault"),
36+
Error("Error"),
37+
LlmFailure("LLMFailure"),
38+
;
39+
40+
override fun toString(): String = resultName
41+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/DocExceptions.kt

Lines changed: 82 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,93 @@
33

44
package software.aws.toolkits.jetbrains.services.amazonqDoc
55

6+
import software.aws.toolkits.jetbrains.services.amazonq.project.RepoSizeError
7+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ClientException
8+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.LlmException
9+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ServiceException
610
import software.aws.toolkits.resources.message
711

8-
open class DocException(
9-
override val message: String?,
10-
override val cause: Throwable? = null,
12+
open class DocClientException(
13+
message: String,
14+
operation: String,
15+
desc: String?,
16+
cause: Throwable? = null,
1117
val remainingIterations: Int? = null,
12-
) : RuntimeException()
18+
) : ClientException(message, operation, desc, cause)
1319

14-
class ZipFileError(override val message: String, override val cause: Throwable?) : RuntimeException()
20+
val denyListedErrors = arrayOf("Deserialization error", "Inaccessible host", "UnknownHost")
21+
fun createUserFacingErrorMessage(message: String?): String? =
22+
if (message != null && denyListedErrors.any { message.contains(it) }) "$FEATURE_NAME API request failed" else message
1523

16-
class CodeIterationLimitError(override val message: String, override val cause: Throwable?) : RuntimeException()
24+
class ReadmeTooLargeException(
25+
operation: String,
26+
desc: String?,
27+
cause: Throwable? = null,
28+
remainingIterations: Int? = null,
29+
) : DocClientException(message("amazonqDoc.exception.readme_too_large"), operation, desc, cause, remainingIterations)
1730

18-
internal fun docServiceError(message: String?, cause: Throwable? = null, remainingIterations: Int? = null): Nothing =
19-
throw DocException(message, cause, remainingIterations)
31+
class ReadmeUpdateTooLargeException(
32+
operation: String,
33+
desc: String?,
34+
cause: Throwable? = null,
35+
remainingIterations: Int? = null,
36+
) : DocClientException(message("amazonqDoc.exception.readme_update_too_large"), operation, desc, cause, remainingIterations)
2037

21-
internal fun conversationIdNotFound(): Nothing =
22-
throw DocException(message("amazonqFeatureDev.exception.conversation_not_found"))
38+
class ContentLengthException(
39+
override val message: String = message("amazonqDoc.exception.content_length_error"),
40+
operation: String,
41+
desc: String?,
42+
cause: Throwable? = null,
43+
) :
44+
RepoSizeError, ClientException(message, operation, desc, cause)
2345

24-
val denyListedErrors = arrayOf("Deserialization error", "Inaccessible host", "UnknownHost")
25-
fun createUserFacingErrorMessage(message: String?): String? =
26-
if (message != null && denyListedErrors.any { message.contains(it) }) "$FEATURE_NAME API request failed" else message
46+
class WorkspaceEmptyException(
47+
operation: String,
48+
desc: String?,
49+
cause: Throwable? = null,
50+
) : DocClientException(message("amazonqDoc.exception.workspace_empty"), operation, desc, cause)
51+
52+
class PromptUnrelatedException(
53+
operation: String,
54+
desc: String?,
55+
cause: Throwable? = null,
56+
remainingIterations: Int? = null,
57+
) : DocClientException(message("amazonqDoc.exception.prompt_unrelated"), operation, desc, cause, remainingIterations)
58+
59+
class PromptTooVagueException(
60+
operation: String,
61+
desc: String?,
62+
cause: Throwable? = null,
63+
remainingIterations: Int? = null,
64+
) : DocClientException(message("amazonqDoc.exception.prompt_too_vague"), operation, desc, cause, remainingIterations)
65+
66+
class PromptRefusalException(
67+
operation: String,
68+
desc: String?,
69+
cause: Throwable? = null,
70+
remainingIterations: Int? = null,
71+
) : DocClientException(message("amazonqFeatureDev.exception.prompt_refusal"), operation, desc, cause, remainingIterations)
72+
73+
class GuardrailsException(
74+
operation: String,
75+
desc: String?,
76+
cause: Throwable? = null,
77+
) : DocClientException(message("amazonqDoc.error_text"), operation, desc, cause)
78+
79+
class NoChangeRequiredException(
80+
operation: String,
81+
desc: String?,
82+
cause: Throwable? = null,
83+
) : DocClientException(message("amazonqDoc.exception.no_change_required"), operation, desc, cause)
84+
85+
class EmptyPatchException(operation: String, desc: String?, cause: Throwable? = null) :
86+
LlmException(message("amazonqDoc.error_text"), operation, desc, cause)
87+
88+
class ThrottlingException(
89+
operation: String,
90+
desc: String?,
91+
cause: Throwable? = null,
92+
) : DocClientException(message("amazonqFeatureDev.exception.throttling"), operation, desc, cause)
93+
94+
class DocGenerationException(operation: String, desc: String?, cause: Throwable? = null) :
95+
ServiceException(message("amazonqDoc.error_text"), operation, desc, cause)

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/controller/DocController.kt

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,11 @@ import software.aws.toolkits.jetbrains.services.amazonq.project.RepoSizeError
3434
import software.aws.toolkits.jetbrains.services.amazonq.toolwindow.AmazonQToolWindowFactory
3535
import software.aws.toolkits.jetbrains.services.amazonqDoc.DEFAULT_RETRY_LIMIT
3636
import software.aws.toolkits.jetbrains.services.amazonqDoc.DIAGRAM_SVG_EXT
37-
import software.aws.toolkits.jetbrains.services.amazonqDoc.DocException
37+
import software.aws.toolkits.jetbrains.services.amazonqDoc.DocClientException
3838
import software.aws.toolkits.jetbrains.services.amazonqDoc.FEATURE_NAME
3939
import software.aws.toolkits.jetbrains.services.amazonqDoc.InboundAppMessagesHandler
40-
import software.aws.toolkits.jetbrains.services.amazonqDoc.ZipFileError
40+
import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataOperationName
41+
import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataResult
4142
import software.aws.toolkits.jetbrains.services.amazonqDoc.cancellingProgressField
4243
import software.aws.toolkits.jetbrains.services.amazonqDoc.createUserFacingErrorMessage
4344
import software.aws.toolkits.jetbrains.services.amazonqDoc.denyListedErrors
@@ -70,6 +71,7 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.storage.ChatSessionSt
7071
import software.aws.toolkits.jetbrains.services.amazonqDoc.util.getFollowUpOptions
7172
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CodeIterationLimitException
7273
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MonthlyConversationLimitError
74+
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ZipFileCorruptedException
7375
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeReferenceGenerated
7476
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
7577
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.NewFileZipInfo
@@ -592,9 +594,12 @@ class DocController(
592594
messenger.sendUpdatePlaceholder(tabId, message("amazonqFeatureDev.placeholder.provide_code_feedback"))
593595
}
594596

595-
private suspend fun processErrorChatMessage(err: Exception, session: DocSession?, tabId: String, isEnableChatInput: Boolean) {
597+
private suspend fun processErrorChatMessage(err: Exception, session: DocSession?, tabId: String) {
596598
logger.warn(err) { "Encountered ${err.message} for tabId: $tabId" }
597599
messenger.sendUpdatePromptProgress(tabId, null)
600+
val docGenerationMode = docGenerationTasks.getTask(tabId).mode
601+
val isEnableChatInput = docGenerationMode == Mode.EDIT &&
602+
(err as? DocClientException)?.remainingIterations?.let { it > 0 } ?: false
598603

599604
when (err) {
600605
is RepoSizeError -> {
@@ -616,7 +621,7 @@ class DocController(
616621
)
617622
}
618623

619-
is ZipFileError -> {
624+
is ZipFileCorruptedException -> {
620625
messenger.sendError(
621626
tabId = tabId,
622627
errMessage = err.message,
@@ -626,12 +631,11 @@ class DocController(
626631
}
627632

628633
is MonthlyConversationLimitError -> {
629-
messenger.sendUpdatePlaceholder(tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_monthly_limit"))
630-
messenger.sendChatInputEnabledMessage(tabId, enabled = true)
631634
messenger.sendMonthlyLimitError(tabId = tabId)
635+
messenger.sendChatInputEnabledMessage(tabId, enabled = false)
632636
}
633637

634-
is DocException -> {
638+
is DocClientException -> {
635639
messenger.sendErrorToUser(
636640
tabId = tabId,
637641
errMessage = err.message,
@@ -765,13 +769,7 @@ class DocController(
765769
)
766770
}
767771
} catch (err: Exception) {
768-
// For non edit mode lock the chat input until they explicitly click one of the follow-ups
769-
var isEnableChatInput = false
770-
if (err is DocException && docGenerationTask.mode == Mode.EDIT) {
771-
isEnableChatInput = err.remainingIterations != null && err.remainingIterations > 0
772-
}
773-
774-
processErrorChatMessage(err, session, tabId, isEnableChatInput)
772+
processErrorChatMessage(err, session, tabId)
775773
}
776774
}
777775

@@ -794,6 +792,10 @@ class DocController(
794792
}
795793

796794
session.send(sessionMessage)
795+
session.sendDocMetricData(
796+
MetricDataOperationName.StartDocGeneration,
797+
MetricDataResult.Success
798+
)
797799

798800
val filePaths: List<NewFileZipInfo> = when (val state = session.sessionState) {
799801
is PrepareDocGenerationState -> state.filePaths ?: emptyList()
@@ -854,7 +856,11 @@ class DocController(
854856
message = IncomingDocMessage.OpenDiff(tabId = followUpMessage.tabId, filePath = filePaths[0].zipFilePath, deleted = false)
855857
)
856858
} catch (err: Exception) {
857-
processErrorChatMessage(err, session, tabId = followUpMessage.tabId, false)
859+
session.sendDocMetricData(
860+
MetricDataOperationName.EndDocGeneration,
861+
session.getMetricResult(err)
862+
)
863+
processErrorChatMessage(err, session, tabId = followUpMessage.tabId)
858864
} finally {
859865
messenger.sendUpdatePlaceholder(
860866
tabId = followUpMessage.tabId,
@@ -873,6 +879,10 @@ class DocController(
873879
messenger.sendChatInputEnabledMessage(tabId = followUpMessage.tabId, enabled = false) // Lock chat input until a follow-up is clicked.
874880
}
875881
}
882+
session.sendDocMetricData(
883+
MetricDataOperationName.EndDocGeneration,
884+
MetricDataResult.Success
885+
)
876886
}
877887

878888
private suspend fun handleEmptyFiles(

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/controller/DocControllerExtensions.kt

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package software.aws.toolkits.jetbrains.services.amazonqDoc.controller
55

66
import com.intellij.notification.NotificationAction
7+
import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataOperationName
8+
import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataResult
79
import software.aws.toolkits.jetbrains.services.amazonqDoc.inProgress
810
import software.aws.toolkits.jetbrains.services.amazonqDoc.messages.DocMessageType
911
import software.aws.toolkits.jetbrains.services.amazonqDoc.messages.FollowUp
@@ -19,7 +21,6 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.messages.sendUpdatePr
1921
import software.aws.toolkits.jetbrains.services.amazonqDoc.session.DocSession
2022
import software.aws.toolkits.jetbrains.services.amazonqDoc.session.PrepareDocGenerationState
2123
import software.aws.toolkits.jetbrains.services.amazonqDoc.util.getFollowUpOptions
22-
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
2324
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeReferenceGenerated
2425
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
2526
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.NewFileZipInfo
@@ -52,6 +53,10 @@ suspend fun DocController.onCodeGeneration(session: DocSession, message: String,
5253
}
5354

5455
val state = session.sessionState
56+
session.sendDocMetricData(
57+
MetricDataOperationName.StartDocGeneration,
58+
MetricDataResult.Success
59+
)
5560

5661
var filePaths: List<NewFileZipInfo> = emptyList()
5762
var deletedFiles: List<DeletedFileInfo> = emptyList()
@@ -119,6 +124,12 @@ suspend fun DocController.onCodeGeneration(session: DocSession, message: String,
119124
messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase))
120125

121126
messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
127+
} catch (err: Exception) {
128+
session.sendDocMetricData(
129+
MetricDataOperationName.EndDocGeneration,
130+
session.getMetricResult(err)
131+
)
132+
throw err
122133
} finally {
123134
messenger.sendAsyncEventProgress(tabId = tabId, inProgress = false) // Finish processing the event
124135
messenger.sendChatInputEnabledMessage(tabId = tabId, enabled = false) // Lock chat input until a follow-up is clicked.
@@ -132,6 +143,10 @@ suspend fun DocController.onCodeGeneration(session: DocSession, message: String,
132143
)
133144
}
134145
}
146+
session.sendDocMetricData(
147+
MetricDataOperationName.EndDocGeneration,
148+
MetricDataResult.Success
149+
)
135150
}
136151

137152
private fun DocController.openChatNotificationAction() = NotificationAction.createSimple(

0 commit comments

Comments
 (0)