telmetry: add ide diagnostics for inline

liumofei-amazon · liumofei-amazon · commit 7f25c1ca35fe · 2025-07-02T16:42:06.000-07:00
diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt
@@ -14,6 +14,7 @@ import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.util.UserDataHolderBase
 import com.intellij.util.concurrency.annotations.RequiresEdt
 import kotlinx.coroutines.channels.Channel
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnection
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.textDocument.InlineCompletionItem
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.textDocument.InlineCompletionListWithReferences
@@ -244,6 +245,7 @@ data class InlineCompletionSessionContext(
     var sessionId: String = "",
     val triggerOffset: Int,
     var counter: Int = 0,
+    val diagnostics: List<IdeDiagnostic>? = emptyList(),
 )
 
 data class InlineCompletionItemContext(
diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/popup/QInlineCompletionProvider.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/popup/QInlineCompletionProvider.kt
@@ -59,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispe
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
 import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
 import software.aws.toolkits.jetbrains.utils.isQConnected
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.CodewhispererTriggerType
@@ -332,6 +333,7 @@ class QInlineCompletionProvider(private val cs: CoroutineScope) : InlineCompleti
                             latencyContext,
                             sessionContext,
                             triggerSessionId,
+                            editor.document
                         )
                         activeTriggerSessions.remove(triggerSessionId)
                     }
@@ -398,6 +400,7 @@ class QInlineCompletionProvider(private val cs: CoroutineScope) : InlineCompleti
         val triggerSessionId = triggerSessionId++
         val latencyContext = LatencyContext(codewhispererEndToEndStart = System.nanoTime())
         val triggerTypeInfo = getTriggerTypeInfo(request)
+        val diagnostics = getDocumentDiagnostics(editor.document, project)
 
         CodeWhispererInvocationStatus.getInstance().setIsInvokingQInline(session, true)
         Disposer.register(session) {
@@ -412,7 +415,7 @@ class QInlineCompletionProvider(private val cs: CoroutineScope) : InlineCompleti
             return InlineCompletionSuggestion.Empty
         }
 
-        val sessionContext = InlineCompletionSessionContext(triggerOffset = request.endOffset)
+        val sessionContext = InlineCompletionSessionContext(triggerOffset = request.endOffset, diagnostics = diagnostics)
 
         // Pagination workaround: Always return exactly 5 variants
         // Create channel placeholder for upcoming pagination results
diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererTelemetryService.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererTelemetryService.kt
@@ -5,11 +5,14 @@ package software.aws.toolkits.jetbrains.services.codewhisperer.telemetry
 
 import com.intellij.openapi.components.Service
 import com.intellij.openapi.components.service
+import com.intellij.openapi.editor.Document
 import com.intellij.openapi.project.Project
 import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import software.aws.toolkits.core.utils.debug
 import software.aws.toolkits.core.utils.getLogger
+import software.aws.toolkits.jetbrains.core.credentials.sono.isInternalUser
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.AmazonQLspService
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.InlineCompletionStates
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.LogInlineCompletionSessionResultsParams
@@ -24,6 +27,10 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispe
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCodeWhispererStartUrl
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getConnectionStartUrl
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
+import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
 import software.aws.toolkits.jetbrains.settings.AwsSettings
 import software.aws.toolkits.telemetry.CodeFixAction
 import software.aws.toolkits.telemetry.CodewhispererCodeScanScope
@@ -75,6 +82,7 @@ class CodeWhispererTelemetryService(private val cs: CoroutineScope) {
         latencyContext: LatencyContext,
         sessionContext: InlineCompletionSessionContext,
         triggerSessionId: Int,
+        document: Document,
     ) {
         if (sessionContext.sessionId.isEmpty()) {
             QInlineCompletionProvider.logInline(triggerSessionId) {
@@ -96,6 +104,18 @@ class CodeWhispererTelemetryService(private val cs: CoroutineScope) {
                 "total session display time: ${CodeWhispererInvocationStatus.getInstance().completionShownTime?.let { Duration.between(it, Instant.now()) }
                     ?.toMillis()?.toDouble()}"
         }
+        var diffDiagnostics = DiagnosticDifferences(
+            added = emptyList(),
+            removed = emptyList()
+        )
+
+        if (isInternalUser(getStartUrl(project))) {
+            val oldDiagnostics = sessionContext.diagnostics.orEmpty()
+            // wait for the IDE itself to update its diagnostics for current file
+            delay(500)
+            val newDiagnostics = getDocumentDiagnostics(document, project)
+            diffDiagnostics = getDiagnosticDifferences(oldDiagnostics, newDiagnostics)
+        }
         val params = LogInlineCompletionSessionResultsParams(
             sessionId = sessionContext.sessionId,
             completionSessionResult = sessionContext.itemContexts.filter { it.item != null }.associate {
@@ -110,7 +130,9 @@ class CodeWhispererTelemetryService(private val cs: CoroutineScope) {
                 ?.toMillis()?.toDouble(),
             // no userInput in JB inline completion API, every new char input will discard the previous trigger so
             // user input is always 0
-            typeaheadLength = 0
+            typeaheadLength = 0,
+            addedDiagnostics = diffDiagnostics.added,
+            removedDiagnostics = diffDiagnostics.removed,
         )
         AmazonQLspService.executeAsyncIfRunning(project) { server ->
             server.logInlineCompletionSessionResults(params)
diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt
@@ -3,12 +3,16 @@
 
 package software.aws.toolkits.jetbrains.services.codewhisperer.util
 
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.lookup.LookupManager
 import com.intellij.ide.BrowserUtil
+import com.intellij.lang.annotation.HighlightSeverity
 import com.intellij.notification.NotificationAction
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.runInEdt
+import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.Editor
+import com.intellij.openapi.editor.impl.DocumentMarkupModel
 import com.intellij.openapi.editor.impl.EditorImpl
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VfsUtil
@@ -21,7 +25,10 @@ import kotlinx.coroutines.Job
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.yield
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
+import software.amazon.awssdk.services.codewhispererruntime.model.Position
+import software.amazon.awssdk.services.codewhispererruntime.model.Range
 import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.credentials.AwsBearerTokenConnection
@@ -347,3 +354,88 @@ object CodeWhispererUtil {
 enum class CaretMovement {
     NO_CHANGE, MOVE_FORWARD, MOVE_BACKWARD
 }
+
+fun getDiagnosticsType(message: String): String {
+    val lowercaseMessage = message.lowercase()
+
+    val diagnosticPatterns = mapOf(
+        "TYPE_ERROR" to listOf("type", "cast"),
+        "SYNTAX_ERROR" to listOf("expected", "indent", "syntax"),
+        "REFERENCE_ERROR" to listOf("undefined", "not defined", "undeclared", "reference", "symbol"),
+        "BEST_PRACTICE" to listOf("deprecated", "unused", "uninitialized", "not initialized"),
+        "SECURITY" to listOf("security", "vulnerability")
+    )
+
+    return diagnosticPatterns
+        .entries
+        .firstOrNull { (_, keywords) ->
+            keywords.any { lowercaseMessage.contains(it) }
+        }
+        ?.key ?: "OTHER"
+}
+
+fun convertSeverity(severity: HighlightSeverity): String = when {
+    severity == HighlightSeverity.ERROR -> "ERROR"
+    severity == HighlightSeverity.WARNING ||
+        severity == HighlightSeverity.WEAK_WARNING -> "WARNING"
+    severity == HighlightSeverity.INFORMATION -> "INFORMATION"
+    severity.toString().contains("TEXT", ignoreCase = true) -> "HINT"
+    severity == HighlightSeverity.INFO -> "INFORMATION"
+    // For severities that might indicate performance issues
+    severity.toString().contains("PERFORMANCE", ignoreCase = true) -> "WARNING"
+    // For deprecation warnings
+    severity.toString().contains("DEPRECATED", ignoreCase = true) -> "WARNING"
+    // Default case
+    else -> "INFORMATION"
+}
+
+fun getDocumentDiagnostics(document: Document, project: Project): List<IdeDiagnostic> = runCatching {
+    DocumentMarkupModel.forDocument(document, project, true)
+        .allHighlighters
+        .mapNotNull { it.errorStripeTooltip as? HighlightInfo }
+        .filter { !it.description.isNullOrEmpty() }
+        .map { info ->
+            val startLine = document.getLineNumber(info.startOffset)
+            val endLine = document.getLineNumber(info.endOffset)
+
+            IdeDiagnostic.builder()
+                .ideDiagnosticType(getDiagnosticsType(info.description))
+                .severity(convertSeverity(info.severity))
+                .source(info.inspectionToolId)
+                .range(
+                    Range.builder()
+                        .start(
+                            Position.builder()
+                                .line(startLine)
+                                .character(document.getLineStartOffset(startLine))
+                                .build()
+                        )
+                        .end(
+                            Position.builder()
+                                .line(endLine)
+                                .character(document.getLineStartOffset(endLine))
+                                .build()
+                        )
+                        .build()
+                )
+                .build()
+        }
+}.getOrElse { e ->
+    getLogger<CodeWhispererUtil>().warn { "Failed to get document diagnostics ${e.message}" }
+    emptyList()
+}
+
+data class DiagnosticDifferences(
+    val added: List<IdeDiagnostic>,
+    val removed: List<IdeDiagnostic>,
+)
+
+fun serializeDiagnostics(diagnostic: IdeDiagnostic): String = "${diagnostic.source()}-${diagnostic.severity()}-${diagnostic.ideDiagnosticType()}"
+
+fun getDiagnosticDifferences(oldDiagnostic: List<IdeDiagnostic>, newDiagnostic: List<IdeDiagnostic>): DiagnosticDifferences {
+    val oldSet = oldDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
+    val newSet = newDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
+    val added = newDiagnostic.filter { i -> !oldSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
+    val removed = oldDiagnostic.filter { i -> !newSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
+    return DiagnosticDifferences(added, removed)
+}
diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/LogInlineCompletionSessionResultsParams.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/LogInlineCompletionSessionResultsParams.kt
@@ -2,11 +2,15 @@
 // SPDX-License-Identifier: Apache-2.0
 
 package software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 
 data class LogInlineCompletionSessionResultsParams(
     val sessionId: String,
     val completionSessionResult: Map<String, InlineCompletionStates>,
     val firstCompletionDisplayLatency: Double?,
     val totalSessionDisplayTime: Double?,
     val typeaheadLength: Long,
+    val addedDiagnostics: List<IdeDiagnostic>? = emptyList(),
+    val removedDiagnostics: List<IdeDiagnostic>? = emptyList(),
+
 )
