Skip to content

Commit 858aacc

Browse files
liumofei-amazonliumofei-amazon
authored
telemetry: add ide diagnostics for inline (#5862)
1 parent 9b7ffec commit 858aacc

File tree

6 files changed

+255
-2
lines changed

6 files changed

+255
-2
lines changed

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import com.intellij.openapi.util.Disposer
1414
import com.intellij.openapi.util.UserDataHolderBase
1515
import com.intellij.util.concurrency.annotations.RequiresEdt
1616
import kotlinx.coroutines.channels.Channel
17+
import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
1718
import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnection
1819
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.textDocument.InlineCompletionItem
1920
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.textDocument.InlineCompletionListWithReferences
@@ -244,6 +245,7 @@ data class InlineCompletionSessionContext(
244245
var sessionId: String = "",
245246
val triggerOffset: Int,
246247
var counter: Int = 0,
248+
val diagnostics: List<IdeDiagnostic>? = emptyList(),
247249
)
248250

249251
data class InlineCompletionItemContext(

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/popup/QInlineCompletionProvider.kt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispe
5959
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
6060
import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
6161
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
62+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
6263
import software.aws.toolkits.jetbrains.utils.isQConnected
6364
import software.aws.toolkits.resources.message
6465
import software.aws.toolkits.telemetry.CodewhispererTriggerType
@@ -332,6 +333,7 @@ class QInlineCompletionProvider(private val cs: CoroutineScope) : InlineCompleti
332333
latencyContext,
333334
sessionContext,
334335
triggerSessionId,
336+
editor.document
335337
)
336338
activeTriggerSessions.remove(triggerSessionId)
337339
}
@@ -398,6 +400,7 @@ class QInlineCompletionProvider(private val cs: CoroutineScope) : InlineCompleti
398400
val triggerSessionId = triggerSessionId++
399401
val latencyContext = LatencyContext(codewhispererEndToEndStart = System.nanoTime())
400402
val triggerTypeInfo = getTriggerTypeInfo(request)
403+
val diagnostics = getDocumentDiagnostics(editor.document, project)
401404

402405
CodeWhispererInvocationStatus.getInstance().setIsInvokingQInline(session, true)
403406
Disposer.register(session) {
@@ -412,7 +415,7 @@ class QInlineCompletionProvider(private val cs: CoroutineScope) : InlineCompleti
412415
return InlineCompletionSuggestion.Empty
413416
}
414417

415-
val sessionContext = InlineCompletionSessionContext(triggerOffset = request.endOffset)
418+
val sessionContext = InlineCompletionSessionContext(triggerOffset = request.endOffset, diagnostics = diagnostics)
416419

417420
// Pagination workaround: Always return exactly 5 variants
418421
// Create channel placeholder for upcoming pagination results

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererTelemetryService.kt

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@ package software.aws.toolkits.jetbrains.services.codewhisperer.telemetry
55

66
import com.intellij.openapi.components.Service
77
import com.intellij.openapi.components.service
8+
import com.intellij.openapi.editor.Document
89
import com.intellij.openapi.project.Project
910
import kotlinx.coroutines.CoroutineScope
11+
import kotlinx.coroutines.delay
1012
import kotlinx.coroutines.launch
1113
import software.aws.toolkits.core.utils.debug
1214
import software.aws.toolkits.core.utils.getLogger
15+
import software.aws.toolkits.jetbrains.core.credentials.sono.isInternalUser
1316
import software.aws.toolkits.jetbrains.services.amazonq.lsp.AmazonQLspService
1417
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.InlineCompletionStates
1518
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.LogInlineCompletionSessionResultsParams
@@ -24,6 +27,10 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispe
2427
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
2528
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCodeWhispererStartUrl
2629
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getConnectionStartUrl
30+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
31+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
32+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
33+
import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
2734
import software.aws.toolkits.jetbrains.settings.AwsSettings
2835
import software.aws.toolkits.telemetry.CodeFixAction
2936
import software.aws.toolkits.telemetry.CodewhispererCodeScanScope
@@ -75,6 +82,7 @@ class CodeWhispererTelemetryService(private val cs: CoroutineScope) {
7582
latencyContext: LatencyContext,
7683
sessionContext: InlineCompletionSessionContext,
7784
triggerSessionId: Int,
85+
document: Document,
7886
) {
7987
if (sessionContext.sessionId.isEmpty()) {
8088
QInlineCompletionProvider.logInline(triggerSessionId) {
@@ -96,6 +104,18 @@ class CodeWhispererTelemetryService(private val cs: CoroutineScope) {
96104
"total session display time: ${CodeWhispererInvocationStatus.getInstance().completionShownTime?.let { Duration.between(it, Instant.now()) }
97105
?.toMillis()?.toDouble()}"
98106
}
107+
var diffDiagnostics = DiagnosticDifferences(
108+
added = emptyList(),
109+
removed = emptyList()
110+
)
111+
112+
if (isInternalUser(getStartUrl(project))) {
113+
val oldDiagnostics = sessionContext.diagnostics.orEmpty()
114+
// wait for the IDE itself to update its diagnostics for current file
115+
delay(500)
116+
val newDiagnostics = getDocumentDiagnostics(document, project)
117+
diffDiagnostics = getDiagnosticDifferences(oldDiagnostics, newDiagnostics)
118+
}
99119
val params = LogInlineCompletionSessionResultsParams(
100120
sessionId = sessionContext.sessionId,
101121
completionSessionResult = sessionContext.itemContexts.filter { it.item != null }.associate {
@@ -110,7 +130,9 @@ class CodeWhispererTelemetryService(private val cs: CoroutineScope) {
110130
?.toMillis()?.toDouble(),
111131
// no userInput in JB inline completion API, every new char input will discard the previous trigger so
112132
// user input is always 0
113-
typeaheadLength = 0
133+
typeaheadLength = 0,
134+
addedDiagnostics = diffDiagnostics.added,
135+
removedDiagnostics = diffDiagnostics.removed,
114136
)
115137
AmazonQLspService.executeAsyncIfRunning(project) { server ->
116138
server.logInlineCompletionSessionResults(params)

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,16 @@
33

44
package software.aws.toolkits.jetbrains.services.codewhisperer.util
55

6+
import com.intellij.codeInsight.daemon.impl.HighlightInfo
67
import com.intellij.codeInsight.lookup.LookupManager
78
import com.intellij.ide.BrowserUtil
9+
import com.intellij.lang.annotation.HighlightSeverity
810
import com.intellij.notification.NotificationAction
911
import com.intellij.openapi.application.ApplicationManager
1012
import com.intellij.openapi.application.runInEdt
13+
import com.intellij.openapi.editor.Document
1114
import com.intellij.openapi.editor.Editor
15+
import com.intellij.openapi.editor.impl.DocumentMarkupModel
1216
import com.intellij.openapi.editor.impl.EditorImpl
1317
import com.intellij.openapi.project.Project
1418
import com.intellij.openapi.vfs.VfsUtil
@@ -21,7 +25,11 @@ import kotlinx.coroutines.Job
2125
import kotlinx.coroutines.delay
2226
import kotlinx.coroutines.launch
2327
import kotlinx.coroutines.yield
28+
import software.amazon.awssdk.services.codewhispererruntime.model.DiagnosticSeverity
29+
import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
2430
import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
31+
import software.amazon.awssdk.services.codewhispererruntime.model.Position
32+
import software.amazon.awssdk.services.codewhispererruntime.model.Range
2533
import software.aws.toolkits.core.utils.getLogger
2634
import software.aws.toolkits.core.utils.warn
2735
import software.aws.toolkits.jetbrains.core.credentials.AwsBearerTokenConnection
@@ -347,3 +355,87 @@ object CodeWhispererUtil {
347355
enum class CaretMovement {
348356
NO_CHANGE, MOVE_FORWARD, MOVE_BACKWARD
349357
}
358+
359+
val diagnosticPatterns = mapOf(
360+
"TYPE_ERROR" to listOf("type", "cast"),
361+
"SYNTAX_ERROR" to listOf("expected", "indent", "syntax"),
362+
"REFERENCE_ERROR" to listOf("undefined", "not defined", "undeclared", "reference", "symbol"),
363+
"BEST_PRACTICE" to listOf("deprecated", "unused", "uninitialized", "not initialized"),
364+
"SECURITY" to listOf("security", "vulnerability")
365+
)
366+
367+
fun getDiagnosticsType(message: String): String {
368+
val lowercaseMessage = message.lowercase()
369+
return diagnosticPatterns
370+
.entries
371+
.firstOrNull { (_, keywords) ->
372+
keywords.any { lowercaseMessage.contains(it) }
373+
}
374+
?.key ?: "OTHER"
375+
}
376+
377+
fun convertSeverity(severity: HighlightSeverity): DiagnosticSeverity = when {
378+
severity == HighlightSeverity.ERROR -> DiagnosticSeverity.ERROR
379+
severity == HighlightSeverity.WARNING ||
380+
severity == HighlightSeverity.WEAK_WARNING -> DiagnosticSeverity.WARNING
381+
severity == HighlightSeverity.INFORMATION -> DiagnosticSeverity.INFORMATION
382+
severity == HighlightSeverity.TEXT_ATTRIBUTES -> DiagnosticSeverity.HINT
383+
severity == HighlightSeverity.INFO -> DiagnosticSeverity.INFORMATION
384+
// For severities that might indicate performance issues
385+
severity.toString().contains("PERFORMANCE", ignoreCase = true) -> DiagnosticSeverity.WARNING
386+
// For deprecation warnings
387+
severity.toString().contains("DEPRECATED", ignoreCase = true) -> DiagnosticSeverity.WARNING
388+
// Default case
389+
else -> DiagnosticSeverity.INFORMATION
390+
}
391+
392+
fun getDocumentDiagnostics(document: Document, project: Project): List<IdeDiagnostic> = runCatching {
393+
DocumentMarkupModel.forDocument(document, project, true)
394+
.allHighlighters
395+
.mapNotNull { it.errorStripeTooltip as? HighlightInfo }
396+
.filter { !it.description.isNullOrEmpty() }
397+
.map { info ->
398+
val startLine = document.getLineNumber(info.startOffset)
399+
val endLine = document.getLineNumber(info.endOffset)
400+
401+
IdeDiagnostic.builder()
402+
.ideDiagnosticType(getDiagnosticsType(info.description))
403+
.severity(convertSeverity(info.severity))
404+
.source(info.inspectionToolId)
405+
.range(
406+
Range.builder()
407+
.start(
408+
Position.builder()
409+
.line(startLine)
410+
.character(document.getLineStartOffset(startLine))
411+
.build()
412+
)
413+
.end(
414+
Position.builder()
415+
.line(endLine)
416+
.character(document.getLineStartOffset(endLine))
417+
.build()
418+
)
419+
.build()
420+
)
421+
.build()
422+
}
423+
}.getOrElse { e ->
424+
getLogger<CodeWhispererUtil>().warn { "Failed to get document diagnostics ${e.message}" }
425+
emptyList()
426+
}
427+
428+
data class DiagnosticDifferences(
429+
val added: List<IdeDiagnostic>,
430+
val removed: List<IdeDiagnostic>,
431+
)
432+
433+
fun serializeDiagnostics(diagnostic: IdeDiagnostic): String = "${diagnostic.source()}-${diagnostic.severity()}-${diagnostic.ideDiagnosticType()}"
434+
435+
fun getDiagnosticDifferences(oldDiagnostic: List<IdeDiagnostic>, newDiagnostic: List<IdeDiagnostic>): DiagnosticDifferences {
436+
val oldSet = oldDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
437+
val newSet = newDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
438+
val added = newDiagnostic.filter { i -> !oldSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
439+
val removed = oldDiagnostic.filter { i -> !newSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
440+
return DiagnosticDifferences(added, removed)
441+
}

‎plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUtilTest.kt

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
package software.aws.toolkits.jetbrains.services.codewhisperer
55

6+
import com.intellij.lang.annotation.HighlightSeverity
67
import com.intellij.openapi.util.SimpleModificationTracker
78
import com.intellij.testFramework.fixtures.CodeInsightTestFixture
89
import kotlinx.coroutines.runBlocking
@@ -12,7 +13,11 @@ import org.junit.Before
1213
import org.junit.Rule
1314
import org.junit.Test
1415
import software.amazon.awssdk.regions.Region
16+
import software.amazon.awssdk.services.codewhispererruntime.model.DiagnosticSeverity
17+
import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
1518
import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
19+
import software.amazon.awssdk.services.codewhispererruntime.model.Position
20+
import software.amazon.awssdk.services.codewhispererruntime.model.Range
1621
import software.amazon.awssdk.services.ssooidc.SsoOidcClient
1722
import software.aws.toolkits.core.utils.test.aStringWithLineCount
1823
import software.aws.toolkits.jetbrains.core.MockClientManagerRule
@@ -23,6 +28,9 @@ import software.aws.toolkits.jetbrains.core.credentials.sono.SONO_URL
2328
import software.aws.toolkits.jetbrains.core.region.MockRegionProviderRule
2429
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCompletionType
2530
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
31+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.convertSeverity
32+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
33+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticsType
2634
import software.aws.toolkits.jetbrains.services.codewhisperer.util.isWithin
2735
import software.aws.toolkits.jetbrains.services.codewhisperer.util.runIfIdcConnectionOrTelemetryEnabled
2836
import software.aws.toolkits.jetbrains.services.codewhisperer.util.toCodeChunk
@@ -263,4 +271,126 @@ class CodeWhispererUtilTest {
263271
val file = fixture.addFileToProject("workspace/projectA1/src/Sample.java", "").virtualFile
264272
assertThat(file.isWithin(projectRoot)).isFalse()
265273
}
274+
275+
@Test
276+
fun `getDiagnosticsType correctly identifies syntax errors`() {
277+
val messages = listOf(
278+
"Expected semicolon at end of line",
279+
"Incorrect indent level",
280+
"Syntax error in expression"
281+
)
282+
283+
messages.forEach { message ->
284+
assertThat(getDiagnosticsType(message)).isEqualTo("SYNTAX_ERROR")
285+
}
286+
}
287+
288+
@Test
289+
fun `getDiagnosticsType correctly identifies type errors`() {
290+
val messages = listOf(
291+
"Cannot cast String to Int",
292+
"Type mismatch: expected String but got Int"
293+
)
294+
295+
messages.forEach { message ->
296+
assertThat(getDiagnosticsType(message)).isEqualTo("TYPE_ERROR")
297+
}
298+
}
299+
300+
@Test
301+
fun `getDiagnosticsType returns OTHER for unrecognized patterns`() {
302+
val message = "Some random message"
303+
assertThat(getDiagnosticsType(message)).isEqualTo("OTHER")
304+
}
305+
306+
@Test
307+
fun `convertSeverity correctly maps severity levels`() {
308+
assertThat(convertSeverity(HighlightSeverity.ERROR)).isEqualTo(DiagnosticSeverity.ERROR)
309+
assertThat(convertSeverity(HighlightSeverity.WARNING)).isEqualTo(DiagnosticSeverity.WARNING)
310+
assertThat(convertSeverity(HighlightSeverity.TEXT_ATTRIBUTES)).isEqualTo(DiagnosticSeverity.HINT)
311+
assertThat(convertSeverity(HighlightSeverity.INFORMATION)).isEqualTo(DiagnosticSeverity.INFORMATION)
312+
assertThat(convertSeverity(HighlightSeverity.INFO)).isEqualTo(DiagnosticSeverity.INFORMATION)
313+
}
314+
315+
@Test
316+
fun `getDiagnosticDifferences correctly identifies added and removed diagnostics`() {
317+
val diagnostic1 = IdeDiagnostic.builder()
318+
.ideDiagnosticType("SYNTAX_ERROR")
319+
.severity("ERROR")
320+
.source("inspection1")
321+
.range(
322+
Range.builder()
323+
.start(Position.builder().line(0).character(0).build())
324+
.end(Position.builder().line(0).character(10).build())
325+
.build()
326+
)
327+
.build()
328+
329+
val diagnostic2 = IdeDiagnostic.builder()
330+
.ideDiagnosticType("TYPE_ERROR")
331+
.severity("WARNING")
332+
.source("inspection2")
333+
.range(
334+
Range.builder()
335+
.start(Position.builder().line(1).character(0).build())
336+
.end(Position.builder().line(1).character(10).build())
337+
.build()
338+
)
339+
.build()
340+
341+
val oldList = listOf(diagnostic1)
342+
val newList = listOf(diagnostic2)
343+
344+
val differences = getDiagnosticDifferences(oldList, newList)
345+
346+
assertThat(differences.added).containsExactly(diagnostic2)
347+
assertThat(differences.removed).containsExactly(diagnostic1)
348+
}
349+
350+
@Test
351+
fun `getDiagnosticDifferences handles empty lists`() {
352+
val diagnostic = IdeDiagnostic.builder()
353+
.ideDiagnosticType("SYNTAX_ERROR")
354+
.severity("ERROR")
355+
.source("inspection1")
356+
.range(
357+
Range.builder()
358+
.start(Position.builder().line(0).character(0).build())
359+
.end(Position.builder().line(0).character(10).build())
360+
.build()
361+
)
362+
.build()
363+
364+
val emptyList = emptyList<IdeDiagnostic>()
365+
val nonEmptyList = listOf(diagnostic)
366+
367+
val differencesWithEmptyOld = getDiagnosticDifferences(emptyList, nonEmptyList)
368+
assertThat(differencesWithEmptyOld.added).containsExactly(diagnostic)
369+
assertThat(differencesWithEmptyOld.removed).isEmpty()
370+
371+
val differencesWithEmptyNew = getDiagnosticDifferences(nonEmptyList, emptyList)
372+
assertThat(differencesWithEmptyNew.added).isEmpty()
373+
assertThat(differencesWithEmptyNew.removed).containsExactly(diagnostic)
374+
}
375+
376+
@Test
377+
fun `getDiagnosticDifferences handles identical lists`() {
378+
val diagnostic = IdeDiagnostic.builder()
379+
.ideDiagnosticType("SYNTAX_ERROR")
380+
.severity("ERROR")
381+
.source("inspection1")
382+
.range(
383+
Range.builder()
384+
.start(Position.builder().line(0).character(0).build())
385+
.end(Position.builder().line(0).character(10).build())
386+
.build()
387+
)
388+
.build()
389+
390+
val list = listOf(diagnostic)
391+
val differences = getDiagnosticDifferences(list, list)
392+
393+
assertThat(differences.added).isEmpty()
394+
assertThat(differences.removed).isEmpty()
395+
}
266396
}

0 commit comments

Comments
 (0)