telemetry(amazonq): Added metric data for /doc and improved error cla…

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/common/clients/AmazonQCodeGenerateClient.kt

@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
 import software.amazon.awssdk.services.codewhispererruntime.model.DocV2AcceptanceEvent
 import software.amazon.awssdk.services.codewhispererruntime.model.DocV2GenerationEvent
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
@@ -89,6 +90,33 @@ class AmazonQCodeGenerateClient(private val project: Project) {
             requestBuilder.userContext(docUserContext)
         }
 
+    fun sendDocMetricData(operationName: String, result: String): SendTelemetryEventResponse =
+        bearerClient().sendTelemetryEvent { requestBuilder ->
+            requestBuilder.telemetryEvent { telemetryEventBuilder ->
+                telemetryEventBuilder.metricData {
+                    it
+                        .metricName("Operation")
+                        .metricValue(1.0)
+                        .timestamp(Instant.now())
+                        .product("DocGeneration")
+                        .dimensions(
+                            listOf(
+                                Dimension.builder()
+                                    .name("operationName")
+                                    .value(operationName)
+                                    .build(),
+                                Dimension.builder()
+                                    .name("result")
+                                    .value(result)
+                                    .build()
+                            )
+                        )
+                }
+            }
+            requestBuilder.optOutPreference(getTelemetryOptOutPreference())
+            requestBuilder.userContext(docUserContext)
+        }
+
     fun createTaskAssistConversation(): CreateTaskAssistConversationResponse = bearerClient().createTaskAssistConversation(
         CreateTaskAssistConversationRequest.builder().build()
     )

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/common/util/AmazonQCodeGenService.kt

@@ -20,7 +20,6 @@ import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.common.clients.AmazonQCodeGenerateClient
 import software.aws.toolkits.jetbrains.common.session.Intent
-import software.aws.toolkits.jetbrains.services.amazonqDoc.docServiceError
 import software.aws.toolkits.jetbrains.services.amazonqDoc.session.DocGenerationStreamResult
 import software.aws.toolkits.jetbrains.services.amazonqDoc.session.ExportDocTaskAssistResultArchiveStreamResult
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ApiException
@@ -34,9 +33,9 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MonthlyConvers
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ServiceException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ZipFileCorruptedException
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
-import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.AmazonqTelemetry
 import software.aws.toolkits.telemetry.MetricResult
+import software.aws.toolkits.jetbrains.services.amazonqDoc.ContentLengthException as DocContentLengthException
 
 class AmazonQCodeGenService(val proxyClient: AmazonQCodeGenerateClient, val project: Project) {
     fun createConversation(): String {
@@ -115,7 +114,7 @@ class AmazonQCodeGenService(val proxyClient: AmazonQCodeGenerateClient, val proj
 
                 if (e is ValidationException && e.message?.contains("Invalid contentLength") == true) {
                     if (featureName?.equals("docGeneration") == true) {
-                        throw docServiceError(message("amazonqDoc.exception.content_length_error"))
+                        throw DocContentLengthException(operation = FeatureDevOperation.CreateUploadUrl.toString(), desc = null, cause = e.cause)
                     }
                     throw ContentLengthException(operation = FeatureDevOperation.CreateUploadUrl.toString(), desc = null, cause = e.cause)
                 }
@@ -162,6 +161,9 @@ class AmazonQCodeGenService(val proxyClient: AmazonQCodeGenerateClient, val proj
                 ) {
                     throw CodeIterationLimitException(operation = FeatureDevOperation.StartTaskAssistCodeGeneration.toString(), desc = null, e.cause)
                 } else if (e is ValidationException && e.message?.contains("repo size is exceeding the limits") == true) {
+                    if (intent == Intent.DOC) {
+                        throw DocContentLengthException(operation = FeatureDevOperation.CreateUploadUrl.toString(), desc = null, cause = e.cause)
+                    }
                     throw ContentLengthException(operation = FeatureDevOperation.StartTaskAssistCodeGeneration.toString(), desc = null, cause = e.cause)
                 } else if (e is ValidationException && e.message?.contains("zipped file is corrupted") == true) {
                     throw ZipFileCorruptedException(operation = FeatureDevOperation.StartTaskAssistCodeGeneration.toString(), desc = null, e.cause)

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/DocConstants.kt

@@ -21,3 +21,21 @@ const val DIAGRAM_SVG_EXT = "svg"
 const val DIAGRAM_DOT_EXT = "dot"
 val SUPPORTED_DIAGRAM_EXT_SET: Set<String> = setOf(DIAGRAM_SVG_EXT, DIAGRAM_DOT_EXT)
 val SUPPORTED_DIAGRAM_FILE_NAME_SET: Set<String> = SUPPORTED_DIAGRAM_EXT_SET.map { INFRA_DIAGRAM_PREFIX + it }.toSet()
+
+enum class MetricDataOperationName(private val operationName: String) {
+    StartDocGeneration("StartDocGeneration"),
+    EndDocGeneration("EndDocGeneration"),
+    ;
+
+    override fun toString(): String = operationName
+}
+
+enum class MetricDataResult(private val resultName: String) {
+    Success("Success"),
+    Fault("Fault"),
+    Error("Error"),
+    LlmFailure("LLMFailure"),
+    ;
+
+    override fun toString(): String = resultName
+}

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/DocExceptions.kt

@@ -3,24 +3,93 @@
 
 package software.aws.toolkits.jetbrains.services.amazonqDoc
 
+import software.aws.toolkits.jetbrains.services.amazonq.project.RepoSizeError
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ClientException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.LlmException
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ServiceException
 import software.aws.toolkits.resources.message
 
-open class DocException(
-    override val message: String?,
-    override val cause: Throwable? = null,
+open class DocClientException(
+    message: String,
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
     val remainingIterations: Int? = null,
-) : RuntimeException()
+) : ClientException(message, operation, desc, cause)
 
-class ZipFileError(override val message: String, override val cause: Throwable?) : RuntimeException()
+val denyListedErrors = arrayOf("Deserialization error", "Inaccessible host", "UnknownHost")
+fun createUserFacingErrorMessage(message: String?): String? =
+    if (message != null && denyListedErrors.any { message.contains(it) }) "$FEATURE_NAME API request failed" else message
 
-class CodeIterationLimitError(override val message: String, override val cause: Throwable?) : RuntimeException()
+class ReadmeTooLargeException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+    remainingIterations: Int? = null,
+) : DocClientException(message("amazonqDoc.exception.readme_too_large"), operation, desc, cause, remainingIterations)
 
-internal fun docServiceError(message: String?, cause: Throwable? = null, remainingIterations: Int? = null): Nothing =
-    throw DocException(message, cause, remainingIterations)
+class ReadmeUpdateTooLargeException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+    remainingIterations: Int? = null,
+) : DocClientException(message("amazonqDoc.exception.readme_update_too_large"), operation, desc, cause, remainingIterations)
 
-internal fun conversationIdNotFound(): Nothing =
-    throw DocException(message("amazonqFeatureDev.exception.conversation_not_found"))
+class ContentLengthException(
+    override val message: String = message("amazonqDoc.exception.content_length_error"),
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+) :
+    RepoSizeError, ClientException(message, operation, desc, cause)
 
-val denyListedErrors = arrayOf("Deserialization error", "Inaccessible host", "UnknownHost")
-fun createUserFacingErrorMessage(message: String?): String? =
-    if (message != null && denyListedErrors.any { message.contains(it) }) "$FEATURE_NAME API request failed" else message
+class WorkspaceEmptyException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+) : DocClientException(message("amazonqDoc.exception.workspace_empty"), operation, desc, cause)
+
+class PromptUnrelatedException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+    remainingIterations: Int? = null,
+) : DocClientException(message("amazonqDoc.exception.prompt_unrelated"), operation, desc, cause, remainingIterations)
+
+class PromptTooVagueException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+    remainingIterations: Int? = null,
+) : DocClientException(message("amazonqDoc.exception.prompt_too_vague"), operation, desc, cause, remainingIterations)
+
+class PromptRefusalException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+    remainingIterations: Int? = null,
+) : DocClientException(message("amazonqFeatureDev.exception.prompt_refusal"), operation, desc, cause, remainingIterations)
+
+class GuardrailsException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+) : DocClientException(message("amazonqDoc.error_text"), operation, desc, cause)
+
+class NoChangeRequiredException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+) : DocClientException(message("amazonqDoc.exception.no_change_required"), operation, desc, cause)
+
+class EmptyPatchException(operation: String, desc: String?, cause: Throwable? = null) :
+    LlmException(message("amazonqDoc.error_text"), operation, desc, cause)
+
+class ThrottlingException(
+    operation: String,
+    desc: String?,
+    cause: Throwable? = null,
+) : DocClientException(message("amazonqFeatureDev.exception.throttling"), operation, desc, cause)
+
+class DocGenerationException(operation: String, desc: String?, cause: Throwable? = null) :
+    ServiceException(message("amazonqDoc.error_text"), operation, desc, cause)

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/controller/DocController.kt

@@ -34,10 +34,11 @@ import software.aws.toolkits.jetbrains.services.amazonq.project.RepoSizeError
 import software.aws.toolkits.jetbrains.services.amazonq.toolwindow.AmazonQToolWindowFactory
 import software.aws.toolkits.jetbrains.services.amazonqDoc.DEFAULT_RETRY_LIMIT
 import software.aws.toolkits.jetbrains.services.amazonqDoc.DIAGRAM_SVG_EXT
-import software.aws.toolkits.jetbrains.services.amazonqDoc.DocException
+import software.aws.toolkits.jetbrains.services.amazonqDoc.DocClientException
 import software.aws.toolkits.jetbrains.services.amazonqDoc.FEATURE_NAME
 import software.aws.toolkits.jetbrains.services.amazonqDoc.InboundAppMessagesHandler
-import software.aws.toolkits.jetbrains.services.amazonqDoc.ZipFileError
+import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataOperationName
+import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataResult
 import software.aws.toolkits.jetbrains.services.amazonqDoc.cancellingProgressField
 import software.aws.toolkits.jetbrains.services.amazonqDoc.createUserFacingErrorMessage
 import software.aws.toolkits.jetbrains.services.amazonqDoc.denyListedErrors
@@ -70,6 +71,7 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.storage.ChatSessionSt
 import software.aws.toolkits.jetbrains.services.amazonqDoc.util.getFollowUpOptions
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.CodeIterationLimitException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.MonthlyConversationLimitError
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.ZipFileCorruptedException
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeReferenceGenerated
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.NewFileZipInfo
@@ -592,9 +594,12 @@ class DocController(
         messenger.sendUpdatePlaceholder(tabId, message("amazonqFeatureDev.placeholder.provide_code_feedback"))
     }
 
-    private suspend fun processErrorChatMessage(err: Exception, session: DocSession?, tabId: String, isEnableChatInput: Boolean) {
+    private suspend fun processErrorChatMessage(err: Exception, session: DocSession?, tabId: String) {
         logger.warn(err) { "Encountered ${err.message} for tabId: $tabId" }
         messenger.sendUpdatePromptProgress(tabId, null)
+        val docGenerationMode = docGenerationTasks.getTask(tabId).mode
+        val isEnableChatInput = docGenerationMode == Mode.EDIT &&
+            (err as? DocClientException)?.remainingIterations?.let { it > 0 } ?: false
 
         when (err) {
             is RepoSizeError -> {
@@ -616,7 +621,7 @@ class DocController(
                 )
             }
 
-            is ZipFileError -> {
+            is ZipFileCorruptedException -> {
                 messenger.sendError(
                     tabId = tabId,
                     errMessage = err.message,
@@ -626,12 +631,11 @@ class DocController(
             }
 
             is MonthlyConversationLimitError -> {
-                messenger.sendUpdatePlaceholder(tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_monthly_limit"))
-                messenger.sendChatInputEnabledMessage(tabId, enabled = true)
                 messenger.sendMonthlyLimitError(tabId = tabId)
+                messenger.sendChatInputEnabledMessage(tabId, enabled = false)
             }
 
-            is DocException -> {
+            is DocClientException -> {
                 messenger.sendErrorToUser(
                     tabId = tabId,
                     errMessage = err.message,
@@ -765,13 +769,7 @@ class DocController(
                 )
             }
         } catch (err: Exception) {
-            // For non edit mode lock the chat input until they explicitly click one of the follow-ups
-            var isEnableChatInput = false
-            if (err is DocException && docGenerationTask.mode == Mode.EDIT) {
-                isEnableChatInput = err.remainingIterations != null && err.remainingIterations > 0
-            }
-
-            processErrorChatMessage(err, session, tabId, isEnableChatInput)
+            processErrorChatMessage(err, session, tabId)
         }
     }
 
@@ -794,6 +792,10 @@ class DocController(
             }
 
             session.send(sessionMessage)
+            session.sendDocMetricData(
+                MetricDataOperationName.StartDocGeneration,
+                MetricDataResult.Success
+            )
 
             val filePaths: List<NewFileZipInfo> = when (val state = session.sessionState) {
                 is PrepareDocGenerationState -> state.filePaths ?: emptyList()
@@ -854,7 +856,11 @@ class DocController(
                 message = IncomingDocMessage.OpenDiff(tabId = followUpMessage.tabId, filePath = filePaths[0].zipFilePath, deleted = false)
             )
         } catch (err: Exception) {
-            processErrorChatMessage(err, session, tabId = followUpMessage.tabId, false)
+            session.sendDocMetricData(
+                MetricDataOperationName.EndDocGeneration,
+                session.getMetricResult(err)
+            )
+            processErrorChatMessage(err, session, tabId = followUpMessage.tabId)
         } finally {
             messenger.sendUpdatePlaceholder(
                 tabId = followUpMessage.tabId,
@@ -873,6 +879,10 @@ class DocController(
                 messenger.sendChatInputEnabledMessage(tabId = followUpMessage.tabId, enabled = false) // Lock chat input until a follow-up is clicked.
             }
         }
+        session.sendDocMetricData(
+            MetricDataOperationName.EndDocGeneration,
+            MetricDataResult.Success
+        )
     }
 
     private suspend fun handleEmptyFiles(

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/controller/DocControllerExtensions.kt

@@ -4,6 +4,8 @@
 package software.aws.toolkits.jetbrains.services.amazonqDoc.controller
 
 import com.intellij.notification.NotificationAction
+import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataOperationName
+import software.aws.toolkits.jetbrains.services.amazonqDoc.MetricDataResult
 import software.aws.toolkits.jetbrains.services.amazonqDoc.inProgress
 import software.aws.toolkits.jetbrains.services.amazonqDoc.messages.DocMessageType
 import software.aws.toolkits.jetbrains.services.amazonqDoc.messages.FollowUp
@@ -19,7 +21,6 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.messages.sendUpdatePr
 import software.aws.toolkits.jetbrains.services.amazonqDoc.session.DocSession
 import software.aws.toolkits.jetbrains.services.amazonqDoc.session.PrepareDocGenerationState
 import software.aws.toolkits.jetbrains.services.amazonqDoc.util.getFollowUpOptions
-import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendSystemPrompt
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.CodeReferenceGenerated
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.DeletedFileInfo
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.NewFileZipInfo
@@ -52,6 +53,10 @@ suspend fun DocController.onCodeGeneration(session: DocSession, message: String,
         }
 
         val state = session.sessionState
+        session.sendDocMetricData(
+            MetricDataOperationName.StartDocGeneration,
+            MetricDataResult.Success
+        )
 
         var filePaths: List<NewFileZipInfo> = emptyList()
         var deletedFiles: List<DeletedFileInfo> = emptyList()
@@ -119,6 +124,12 @@ suspend fun DocController.onCodeGeneration(session: DocSession, message: String,
         messenger.sendSystemPrompt(tabId = tabId, followUp = getFollowUpOptions(session.sessionState.phase))
 
         messenger.sendUpdatePlaceholder(tabId = tabId, newPlaceholder = message("amazonqFeatureDev.placeholder.after_code_generation"))
+    } catch (err: Exception) {
+        session.sendDocMetricData(
+            MetricDataOperationName.EndDocGeneration,
+            session.getMetricResult(err)
+        )
+        throw err
     } finally {
         messenger.sendAsyncEventProgress(tabId = tabId, inProgress = false) // Finish processing the event
         messenger.sendChatInputEnabledMessage(tabId = tabId, enabled = false) // Lock chat input until a follow-up is clicked.
@@ -132,6 +143,10 @@ suspend fun DocController.onCodeGeneration(session: DocSession, message: String,
             )
         }
     }
+    session.sendDocMetricData(
+        MetricDataOperationName.EndDocGeneration,
+        MetricDataResult.Success
+    )
 }
 
 private fun DocController.openChatNotificationAction() = NotificationAction.createSimple(