telemetry(amazonq): Add changed IDE diagnostics after user acceptance (#5613)

When user accepts suggestion from inline completion or chat, there can be a change in the current open editor's IDE diagnostics, this can be used as a measure for code suggestion quality. Ref aws/aws-toolkit-vscode#7130.

This change is part of the server side workspace context.

To reduce the risk of large blast radius, this change is only applied for users who are in the experiment of server side project context (both treatment and control), which is the Amzn idc users. 

Verified backend request id 6faef702-938e-42eb-894a-957d9757186e

Author: leigaol

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt

@@ -21,6 +21,7 @@ import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.options.ShowSettingsUtil
 import com.intellij.psi.PsiDocumentManager
 import kotlinx.coroutines.cancelChildren
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.catch
 import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.first
@@ -36,6 +37,7 @@ import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.core.utils.info
 import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.coroutines.EDT
+import software.aws.toolkits.jetbrains.core.credentials.sono.isInternalUser
 import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitContext
 import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
 import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededState
@@ -49,6 +51,8 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhisp
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererUserModificationTracker
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
 import software.aws.toolkits.jetbrains.services.cwc.InboundAppMessagesHandler
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
@@ -214,6 +218,7 @@ class ChatController private constructor(
             val caret: Caret = editor.caretModel.primaryCaret
             val offset: Int = caret.offset
 
+            val oldDiagnostics = getDocumentDiagnostics(editor.document, context.project)
             ApplicationManager.getApplication().runWriteAction {
                 WriteCommandAction.runWriteCommandAction(context.project) {
                     if (caret.hasSelection()) {
@@ -236,6 +241,12 @@ class ChatController private constructor(
                     )
                 }
             }
+            if (isInternalUser(getStartUrl(context.project))) {
+                // wait for the IDE itself to update its diagnostics for current file
+                delay(500)
+                val newDiagnostics = getDocumentDiagnostics(editor.document, context.project)
+                message.diagnosticsDifferences = getDiagnosticDifferences(oldDiagnostics, newDiagnostics)
+            }
         }
         telemetryHelper.recordInteractWithMessage(message)
 

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt

@@ -270,6 +270,8 @@ class TelemetryHelper(private val project: Project, private val sessionStorage:
                     acceptedCharacterCount(message.code.length)
                     acceptedLineCount(message.code.lines().size)
                     hasProjectLevelContext(getMessageHasProjectContext(message.messageId))
+                    addedIdeDiagnostics(message.diagnosticsDifferences?.added)
+                    removedIdeDiagnostics(message.diagnosticsDifferences?.removed)
                 }.build()
             }
 

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/messages/CwcMessage.kt

@@ -18,6 +18,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthFollowUpType
 import software.aws.toolkits.jetbrains.services.amazonq.messages.AmazonQMessage
 import software.aws.toolkits.jetbrains.services.amazonq.onboarding.OnboardingPageInteractionType
 import software.aws.toolkits.jetbrains.services.amazonq.util.HighlightCommand
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.FollowUpType
 import java.time.Instant
 
@@ -95,6 +96,7 @@ sealed interface IncomingCwcMessage : CwcMessage {
         val codeBlockIndex: Int?,
         val totalCodeBlocks: Int?,
         val codeBlockLanguage: String?,
+        var diagnosticsDifferences: DiagnosticDifferences?,
     ) : IncomingCwcMessage, TabId, MessageId
 
     data class TriggerTabIdReceived(

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt

@@ -44,6 +44,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
 import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererCustomization
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.ChatSession
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.CodeNamesImpl
@@ -475,6 +476,7 @@ class TelemetryHelperTest {
         val inserTionTargetType = "insertionTargetType"
         val eventId = "eventId"
         val code = "println()"
+        val diagnosticDifferences = DiagnosticDifferences(emptyList(), emptyList())
 
         sut.recordInteractWithMessage(
             IncomingCwcMessage.InsertCodeAtCursorPosition(
@@ -487,7 +489,8 @@ class TelemetryHelperTest {
                 eventId,
                 codeBlockIndex,
                 totalCodeBlocks,
-                lang
+                lang,
+                diagnosticDifferences
             )
         )
 
@@ -503,6 +506,8 @@ class TelemetryHelperTest {
                     acceptedLineCount(code.lines().size)
                     customizationArn(customizationArn)
                     hasProjectLevelContext(false)
+                    addedIdeDiagnostics(emptyList())
+                    removedIdeDiagnostics(emptyList())
                 }.build()
             )
         )

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt

@@ -37,6 +37,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.TargetCode
 import software.amazon.awssdk.services.codewhispererruntime.model.UserIntent
 import software.aws.toolkits.core.utils.debug
 import software.aws.toolkits.core.utils.getLogger
+import software.aws.toolkits.jetbrains.core.credentials.sono.isInternalUser
 import software.aws.toolkits.jetbrains.services.amazonq.codeWhispererUserContext
 import software.aws.toolkits.jetbrains.services.amazonq.profile.QRegionProfileManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererCustomization
@@ -47,6 +48,10 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestCon
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.ResponseContext
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
+import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
 import software.aws.toolkits.telemetry.CodewhispererCompletionType
 import software.aws.toolkits.telemetry.CodewhispererSuggestionState
 import java.time.Instant
@@ -340,7 +345,17 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         ) {
             e2eLatency = 0.0
         }
-
+        var diffDiagnostics = DiagnosticDifferences(
+            added = emptyList(),
+            removed = emptyList()
+        )
+        if (suggestionState == CodewhispererSuggestionState.Accept && isInternalUser(getStartUrl(project))) {
+            val oldDiagnostics = requestContext.diagnostics.orEmpty()
+            // wait for the IDE itself to update its diagnostics for current file
+            Thread.sleep(500)
+            val newDiagnostics = getDocumentDiagnostics(requestContext.editor.document, project)
+            diffDiagnostics = getDiagnosticDifferences(oldDiagnostics, newDiagnostics)
+        }
         return bearerClient().sendTelemetryEvent { requestBuilder ->
             requestBuilder.telemetryEvent { telemetryEventBuilder ->
                 telemetryEventBuilder.userTriggerDecisionEvent {
@@ -358,6 +373,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                     it.customizationArn(requestContext.customizationArn.nullize(nullizeSpaces = true))
                     it.numberOfRecommendations(numberOfRecommendations)
                     it.acceptedCharacterCount(acceptedCharCount)
+                    it.addedIdeDiagnostics(diffDiagnostics.added)
+                    it.removedIdeDiagnostics(diffDiagnostics.removed)
                 }
             }
             requestBuilder.optOutPreference(getTelemetryOptOutPreference())

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt

@@ -36,6 +36,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.Completion
 import software.amazon.awssdk.services.codewhispererruntime.model.FileContext
 import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsRequest
 import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 import software.amazon.awssdk.services.codewhispererruntime.model.ProgrammingLanguage
 import software.amazon.awssdk.services.codewhispererruntime.model.RecommendationsWithReferencesPreference
 import software.amazon.awssdk.services.codewhispererruntime.model.ResourceNotFoundException
@@ -87,6 +88,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhisperer
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.notifyErrorCodeWhispererUsageLimit
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.promptReAuth
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.FileContextProvider
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
 import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings
 import software.aws.toolkits.jetbrains.utils.isInjectedText
 import software.aws.toolkits.jetbrains.utils.isQExpired
@@ -691,6 +693,7 @@ class CodeWhispererService(private val cs: CoroutineScope) : Disposable {
         } catch (e: Exception) {
             LOG.warn { "Cannot get workspaceId from LSP'$e'" }
         }
+        val diagnostics = getDocumentDiagnostics(editor.document, project)
         return RequestContext(
             project,
             editor,
@@ -703,6 +706,7 @@ class CodeWhispererService(private val cs: CoroutineScope) : Disposable {
             customizationArn,
             profileArn,
             workspaceId,
+            diagnostics
         )
     }
 
@@ -895,6 +899,7 @@ data class RequestContext(
     val customizationArn: String?,
     val profileArn: String?,
     val workspaceId: String?,
+    val diagnostics: List<IdeDiagnostic>?,
 ) {
     // TODO: should make the entire getRequestContext() suspend function instead of making supplemental context only
     var supplementalContext: SupplementalContextInfo? = null

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt

@@ -3,12 +3,16 @@
 
 package software.aws.toolkits.jetbrains.services.codewhisperer.util
 
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.lookup.LookupManager
 import com.intellij.ide.BrowserUtil
+import com.intellij.lang.annotation.HighlightSeverity
 import com.intellij.notification.NotificationAction
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.runInEdt
+import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.Editor
+import com.intellij.openapi.editor.impl.DocumentMarkupModel
 import com.intellij.openapi.editor.impl.EditorImpl
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VfsUtil
@@ -22,7 +26,10 @@ import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.yield
 import software.amazon.awssdk.services.codewhispererruntime.model.Completion
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
+import software.amazon.awssdk.services.codewhispererruntime.model.Position
+import software.amazon.awssdk.services.codewhispererruntime.model.Range
 import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.credentials.AwsBearerTokenConnection
@@ -361,3 +368,88 @@ object CodeWhispererUtil {
 enum class CaretMovement {
     NO_CHANGE, MOVE_FORWARD, MOVE_BACKWARD
 }
+
+fun getDiagnosticsType(message: String): String {
+    val lowercaseMessage = message.lowercase()
+
+    val diagnosticPatterns = mapOf(
+        "TYPE_ERROR" to listOf("type", "cast"),
+        "SYNTAX_ERROR" to listOf("expected", "indent", "syntax"),
+        "REFERENCE_ERROR" to listOf("undefined", "not defined", "undeclared", "reference", "symbol"),
+        "BEST_PRACTICE" to listOf("deprecated", "unused", "uninitialized", "not initialized"),
+        "SECURITY" to listOf("security", "vulnerability")
+    )
+
+    return diagnosticPatterns
+        .entries
+        .firstOrNull { (_, keywords) ->
+            keywords.any { lowercaseMessage.contains(it) }
+        }
+        ?.key ?: "OTHER"
+}
+
+fun convertSeverity(severity: HighlightSeverity): String = when {
+    severity == HighlightSeverity.ERROR -> "ERROR"
+    severity == HighlightSeverity.WARNING ||
+        severity == HighlightSeverity.WEAK_WARNING -> "WARNING"
+    severity == HighlightSeverity.INFORMATION -> "INFORMATION"
+    severity.toString().contains("TEXT", ignoreCase = true) -> "HINT"
+    severity == HighlightSeverity.INFO -> "INFORMATION"
+    // For severities that might indicate performance issues
+    severity.toString().contains("PERFORMANCE", ignoreCase = true) -> "WARNING"
+    // For deprecation warnings
+    severity.toString().contains("DEPRECATED", ignoreCase = true) -> "WARNING"
+    // Default case
+    else -> "INFORMATION"
+}
+
+fun getDocumentDiagnostics(document: Document, project: Project): List<IdeDiagnostic> = runCatching {
+    DocumentMarkupModel.forDocument(document, project, true)
+        .allHighlighters
+        .mapNotNull { it.errorStripeTooltip as? HighlightInfo }
+        .filter { !it.description.isNullOrEmpty() }
+        .map { info ->
+            val startLine = document.getLineNumber(info.startOffset)
+            val endLine = document.getLineNumber(info.endOffset)
+
+            IdeDiagnostic.builder()
+                .ideDiagnosticType(getDiagnosticsType(info.description))
+                .severity(convertSeverity(info.severity))
+                .source(info.inspectionToolId)
+                .range(
+                    Range.builder()
+                        .start(
+                            Position.builder()
+                                .line(startLine)
+                                .character(document.getLineStartOffset(startLine))
+                                .build()
+                        )
+                        .end(
+                            Position.builder()
+                                .line(endLine)
+                                .character(document.getLineStartOffset(endLine))
+                                .build()
+                        )
+                        .build()
+                )
+                .build()
+        }
+}.getOrElse { e ->
+    getLogger<CodeWhispererUtil>().warn { "Failed to get document diagnostics ${e.message}" }
+    emptyList()
+}
+
+data class DiagnosticDifferences(
+    val added: List<IdeDiagnostic>,
+    val removed: List<IdeDiagnostic>,
+)
+
+fun serializeDiagnostics(diagnostic: IdeDiagnostic): String = "${diagnostic.source()}-${diagnostic.severity()}-${diagnostic.ideDiagnosticType()}"
+
+fun getDiagnosticDifferences(oldDiagnostic: List<IdeDiagnostic>, newDiagnostic: List<IdeDiagnostic>): DiagnosticDifferences {
+    val oldSet = oldDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
+    val newSet = newDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
+    val added = newDiagnostic.filter { i -> !oldSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
+    val removed = oldDiagnostic.filter { i -> !newSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
+    return DiagnosticDifferences(added, removed)
+}

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt

@@ -179,6 +179,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
             aString(),
             aString(),
             aString(),
+            emptyList()
         )
         val responseContext = ResponseContext("sessionId")
         val recommendationContext = RecommendationContext(

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererServiceTest.kt

@@ -215,6 +215,7 @@ class CodeWhispererServiceTest {
                 customizationArn = "fake-arn",
                 profileArn = "fake-arn",
                 workspaceId = null,
+                diagnostics = emptyList()
             )
         )