Skip to content

Commit bf33e46

Browse files
laileni-awslaileni-aws
authored
telemetry(amazonq): Adding cwsprChatProgrammingLanguage parameter to insertAtCursor and CopyAtClipboard events (aws#4959)
* Adding Acceptance metrics for UTG
1 parent dd00e96 commit bf33e46

File tree

11 files changed

+109
-73
lines changed

11 files changed

+109
-73
lines changed

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ mockitoKotlin = "5.4.0"
2525
mockk = "1.13.10"
2626
nimbus-jose-jwt = "9.40"
2727
node-gradle = "7.0.2"
28-
telemetryGenerator = "1.0.262"
28+
telemetryGenerator = "1.0.272"
2929
testLogger = "4.0.0"
3030
testRetry = "1.5.10"
3131
# test-only; platform provides slf4j transitively at runtime

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import kotlinx.coroutines.flow.onStart
1111
import software.amazon.awssdk.awscore.exception.AwsServiceException
1212
import software.amazon.awssdk.services.codewhispererstreaming.model.CodeWhispererStreamingException
1313
import software.aws.toolkits.core.utils.convertMarkdownToHTML
14+
import software.aws.toolkits.core.utils.extractCodeBlockLanguage
1415
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
1516
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
1617
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatResponseEvent
@@ -33,6 +34,8 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
3334
private val codeReferences = mutableListOf<CodeReference>()
3435
private var requestId: String = ""
3536
private var statusCode: Int = 0
37+
private val defaultTestGenResponseLanguage: String = "plaintext"
38+
private var codeBlockLanguage: String = defaultTestGenResponseLanguage
3639

3740
companion object {
3841
private val CODE_BLOCK_PATTERN = Regex("<pre>\\s*<code")
@@ -209,6 +212,10 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
209212
} else {
210213
responseText.toString()
211214
}
215+
if (codeBlockLanguage == defaultTestGenResponseLanguage) {
216+
// To get the language of generated code in Q chat.
217+
codeBlockLanguage = extractCodeBlockLanguage(message)
218+
}
212219
ChatMessage(
213220
tabId = tabId,
214221
triggerId = triggerId,
@@ -217,6 +224,7 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
217224
message = message,
218225
codeReference = codeReferences,
219226
userIntent = data.userIntent,
227+
codeBlockLanguage = codeBlockLanguage,
220228
)
221229
} else {
222230
null

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,8 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
219219
credentialStartUrl = getStartUrl(context.project),
220220
cwsprChatCodeBlockIndex = message.codeBlockIndex?.toLong(),
221221
cwsprChatTotalCodeBlocks = message.totalCodeBlocks?.toLong(),
222-
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
222+
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId),
223+
cwsprChatProgrammingLanguage = message.codeBlockLanguage,
223224
)
224225
ChatInteractWithMessageEvent.builder().apply {
225226
conversationId(getConversationId(message.tabId).orEmpty())
@@ -244,7 +245,8 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
244245
credentialStartUrl = getStartUrl(context.project),
245246
cwsprChatCodeBlockIndex = message.codeBlockIndex?.toLong(),
246247
cwsprChatTotalCodeBlocks = message.totalCodeBlocks?.toLong(),
247-
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
248+
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId),
249+
cwsprChatProgrammingLanguage = message.codeBlockLanguage,
248250
)
249251
ChatInteractWithMessageEvent.builder().apply {
250252
conversationId(getConversationId(message.tabId).orEmpty())

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/messages/CwcMessage.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ sealed interface IncomingCwcMessage : CwcMessage {
7272
val eventId: String?,
7373
val codeBlockIndex: Int?,
7474
val totalCodeBlocks: Int?,
75+
val codeBlockLanguage: String?,
7576
) : IncomingCwcMessage
7677

7778
data class InsertCodeAtCursorPosition(
@@ -84,6 +85,7 @@ sealed interface IncomingCwcMessage : CwcMessage {
8485
val eventId: String?,
8586
val codeBlockIndex: Int?,
8687
val totalCodeBlocks: Int?,
88+
val codeBlockLanguage: String?,
8789
) : IncomingCwcMessage
8890

8991
data class TriggerTabIdReceived(
@@ -214,6 +216,7 @@ data class ChatMessage(
214216
val relatedSuggestions: List<Suggestion>? = null,
215217
val codeReference: List<CodeReference>? = null,
216218
val userIntent: UserIntent? = null,
219+
val codeBlockLanguage: String? = "plaintext",
217220
) : UiMessage(
218221
tabId = tabId,
219222
type = "chatMessage",

‎plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,8 @@ class TelemetryHelperTest {
420420
"insertionTargetType",
421421
"eventId",
422422
codeBlockIndex,
423-
totalCodeBlocks
423+
totalCodeBlocks,
424+
lang
424425
)
425426
)
426427

@@ -484,7 +485,8 @@ class TelemetryHelperTest {
484485
emptyList(),
485486
eventId,
486487
codeBlockIndex,
487-
totalCodeBlocks
488+
totalCodeBlocks,
489+
lang
488490
)
489491
)
490492

‎plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/cwChatConnector.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ export class Connector {
101101
codeBlockIndex?: number,
102102
totalCodeBlocks?: number,
103103
userIntent?: string,
104+
codeBlockLanguage?: string,
104105
): void => {
105106
this.sendMessageToExtension({
106107
tabID: tabID,
@@ -113,7 +114,8 @@ export class Connector {
113114
eventId,
114115
codeBlockIndex,
115116
totalCodeBlocks,
116-
userIntent
117+
userIntent,
118+
codeBlockLanguage
117119
})
118120
}
119121

@@ -127,6 +129,7 @@ export class Connector {
127129
codeBlockIndex?: number,
128130
totalCodeBlocks?: number,
129131
userIntent?: string,
132+
codeBlockLanguage?: string,
130133
): void => {
131134
this.sendMessageToExtension({
132135
tabID: tabID,
@@ -139,7 +142,8 @@ export class Connector {
139142
eventId,
140143
codeBlockIndex,
141144
totalCodeBlocks,
142-
userIntent
145+
userIntent,
146+
codeBlockLanguage
143147
})
144148
}
145149

@@ -271,6 +275,7 @@ export class Connector {
271275
canBeVoted: true,
272276
codeReference: messageData.codeReference,
273277
userIntent: messageData.userIntent,
278+
codeBlockLanguage: messageData.codeBlockLanguage,
274279
}
275280

276281
// If it is not there we will not set it
@@ -304,6 +309,7 @@ export class Connector {
304309
messageId: messageData.messageId,
305310
codeReference: messageData.codeReference,
306311
userIntent: messageData.userIntent,
312+
codeBlockLanguage: messageData.codeBlockLanguage,
307313
followUp:
308314
messageData.followUps !== undefined && messageData.followUps.length > 0
309315
? {

‎plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ export interface ChatPayload {
3131
}
3232

3333
export interface CWCChatItem extends ChatItem {
34-
userIntent?: string
34+
userIntent?: string,
35+
codeBlockLanguage?: string,
3536
}
3637

3738
export interface ConnectorProps {
@@ -235,7 +236,8 @@ export class Connector {
235236
eventId?: string,
236237
codeBlockIndex?: number,
237238
totalCodeBlocks?: number,
238-
userIntent?: string
239+
userIntent?: string,
240+
codeBlockLanguage?: string
239241
): void => {
240242
switch (this.tabsStorage.getTab(tabID)?.type) {
241243
case 'cwc':
@@ -248,7 +250,8 @@ export class Connector {
248250
eventId,
249251
codeBlockIndex,
250252
totalCodeBlocks,
251-
userIntent
253+
userIntent,
254+
codeBlockLanguage
252255
)
253256
break
254257
case 'featuredev':
@@ -266,7 +269,8 @@ export class Connector {
266269
eventId?: string,
267270
codeBlockIndex?: number,
268271
totalCodeBlocks?: number,
269-
userIntent?: string
272+
userIntent?: string,
273+
codeBlockLanguage?: string
270274
): void => {
271275
switch (this.tabsStorage.getTab(tabID)?.type) {
272276
case 'cwc':
@@ -279,7 +283,8 @@ export class Connector {
279283
eventId,
280284
codeBlockIndex,
281285
totalCodeBlocks,
282-
userIntent
286+
userIntent,
287+
codeBlockLanguage
283288
)
284289
break
285290
case 'featuredev':

‎plugins/amazonq/mynah-ui/src/mynah-ui/ui/main.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
2424
let mynahUI: MynahUI
2525
// eslint-disable-next-line prefer-const
2626
let connector: Connector
27-
const messageUserIntentMap = new Map<string, string>()
27+
const responseMetadata = new Map<string, string[]>()
2828

2929
const tabsStorage = new TabsStorage({
3030
onTabTimeout: tabID => {
@@ -254,8 +254,8 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
254254
? { type: ChatItemType.CODE_RESULT, fileList: item.fileList }
255255
: {}),
256256
})
257-
if (item.messageId !== undefined && item.userIntent !== undefined) {
258-
messageUserIntentMap.set(item.messageId, item.userIntent)
257+
if (item.messageId !== undefined && item.userIntent !== undefined && item.codeBlockLanguage !== undefined) {
258+
responseMetadata.set(item.messageId, [item.userIntent, item.codeBlockLanguage])
259259
}
260260
return
261261
}
@@ -466,7 +466,8 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
466466
eventId,
467467
codeBlockIndex,
468468
totalCodeBlocks,
469-
messageUserIntentMap.get(messageId) ?? undefined
469+
responseMetadata.get(messageId)?.[0] ?? undefined,
470+
responseMetadata.get(messageId)?.[1] ?? undefined
470471
)
471472
break
472473
case 'copy':
@@ -479,7 +480,8 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
479480
eventId,
480481
codeBlockIndex,
481482
totalCodeBlocks,
482-
messageUserIntentMap.get(messageId) ?? undefined
483+
responseMetadata.get(messageId)?.[0] ?? undefined,
484+
responseMetadata.get(messageId)?.[1] ?? undefined
483485
)
484486
mynahUI.notify({
485487
type: NotificationType.SUCCESS,

‎plugins/core/core/src/software/aws/toolkits/core/utils/TextUtils.kt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,25 @@ fun convertMarkdownToHTML(markdown: String): String {
1717
return htmlRenderer.render(document)
1818
}
1919

20+
fun extractCodeBlockLanguage(message: String): String {
21+
// This fulfills both the cases of unit test generation(java, python) and general use case(Non java and Non python) languages.
22+
val defaultTestGenResponseLanguage = "plaintext"
23+
val indexStart = 3
24+
val codeBlockStart = message.indexOf("```")
25+
if (codeBlockStart == -1) {
26+
return defaultTestGenResponseLanguage
27+
}
28+
29+
val languageStart = codeBlockStart + indexStart
30+
val languageEnd = message.indexOf('\n', languageStart)
31+
32+
if (languageEnd == -1) {
33+
return defaultTestGenResponseLanguage
34+
}
35+
36+
return message.substring(languageStart, languageEnd).trim().ifEmpty { defaultTestGenResponseLanguage }
37+
}
38+
2039
class CodeBlockRenderer(private val html: HtmlWriter) : NodeRenderer {
2140
override fun getNodeTypes(): Set<Class<out Node>> = setOf(FencedCodeBlock::class.java)
2241
override fun render(node: Node?) {

‎plugins/core/jetbrains-community/resources/telemetryOverride.json‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,6 @@
9999
"type": "int",
100100
"description": "Cpu usage of LSP server"
101101
},
102-
{
103-
"name": "cwsprChatProgrammingLanguage",
104-
"type": "string",
105-
"description": "Programming language associated with the message"
106-
},
107102
{
108103
"name": "cwsprChatConversationType",
109104
"type": "string",
@@ -644,57 +639,6 @@
644639
}
645640
]
646641
},
647-
{
648-
"name": "amazonq_interactWithMessage",
649-
"description": "When a user interacts with a message in the conversation",
650-
"metadata": [
651-
{
652-
"type": "cwsprChatConversationId"
653-
},
654-
{
655-
"type": "cwsprChatMessageId"
656-
},
657-
{
658-
"type": "cwsprChatUserIntent",
659-
"required": false
660-
},
661-
{
662-
"type": "cwsprChatInteractionType"
663-
},
664-
{
665-
"type": "cwsprChatInteractionTarget",
666-
"required": false
667-
},
668-
{
669-
"type": "cwsprChatCodeBlockIndex",
670-
"required": false
671-
},
672-
{
673-
"type": "cwsprChatTotalCodeBlocks",
674-
"required": false
675-
},
676-
{
677-
"type": "cwsprChatAcceptedCharactersLength",
678-
"required": false
679-
},
680-
{
681-
"type": "cwsprChatAcceptedNumberOfLines",
682-
"required": false
683-
},
684-
{
685-
"type": "cwsprChatHasReference",
686-
"required": false
687-
},
688-
{
689-
"type": "credentialStartUrl",
690-
"required": false
691-
},
692-
{
693-
"type": "cwsprChatHasProjectContext",
694-
"required": false
695-
}
696-
]
697-
},
698642
{
699643
"name": "amazonq_modifyCode",
700644
"description": "Percentage of code modified by the user after copying/inserting code from a message",

0 commit comments

Comments
 (0)