Skip to content

fix(amazonq): interaction event project context telemetry boolean accuracy #4883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Sep 18, 2024
Merged

fix(amazonq): interaction event project context telemetry boolean accuracy #4883

Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
a1a3331
fix interaction event project context telemetry bug
leigaol Sep 16, 2024
8f49fcd
fix unit test
leigaol Sep 16, 2024
b6cf377
simplify changes
leigaol Sep 16, 2024
503b027
debug message id
leigaol Sep 16, 2024
ed3b0e3
fix the message id in interaction events
leigaol Sep 17, 2024
61531ac
Merge branch 'inter_event_msg_id' of github.com:leigaol/aws-toolkit-j…
leigaol Sep 17, 2024
65b1df1
fix
leigaol Sep 17, 2024
a5a3eae
update
leigaol Sep 17, 2024
f1bc519
minimize changes
leigaol Sep 17, 2024
8e5acdc
Merge branch 'main' into wks_tele_bug
leigaol Sep 17, 2024
c46fea1
fix build unit test
leigaol Sep 17, 2024
2487d49
Merge branch 'wks_tele_bug' of github.com:leigaol/aws-toolkit-jetbrai…
leigaol Sep 17, 2024
bec7d12
fix test
leigaol Sep 17, 2024
2db5033
Merge branch 'main' into wks_tele_bug
leigaol Sep 17, 2024
98c5bb1
Merge branch 'main' into wks_tele_bug
leigaol Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions ...software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
private val responseStreamStartTime: MutableMap<String, Instant> = mutableMapOf()
private val responseStreamTotalTime: MutableMap<String, Int> = mutableMapOf()
private val responseStreamTimeForChunks: MutableMap<String, MutableList<Instant>> = mutableMapOf()
private val responseHasProjectContext: MutableMap<String, Boolean> = mutableMapOf()

private val customization: CodeWhispererCustomization?
get() = CodeWhispererModelConfigurator.getInstance().activeCustomization(context.project)
Expand Down Expand Up @@ -94,6 +95,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se

// When Chat API responds to a user message (full response streamed)
fun recordAddMessage(data: ChatRequestData, response: ChatMessage, responseLength: Int, statusCode: Int, numberOfCodeBlocks: Int) {
val hasProjectContext = getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty()
AmazonqTelemetry.addMessage(
cwsprChatConversationId = getConversationId(response.tabId).orEmpty(),
cwsprChatMessageId = response.messageId,
Expand All @@ -116,9 +118,9 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
cwsprChatConversationType = CwsprChatConversationType.Chat,
credentialStartUrl = getStartUrl(context.project),
codewhispererCustomizationArn = data.customization?.arn,
cwsprChatHasProjectContext = getIsProjectContextEnabled() && data.useRelevantDocuments && data.relevantTextDocuments.isNotEmpty()
cwsprChatHasProjectContext = hasProjectContext
)

responseHasProjectContext[response.messageId] = hasProjectContext
val programmingLanguage = data.activeFileContext.fileContext?.fileLanguage
val validProgrammingLanguage = if (ChatSessionV1.validLanguages.contains(programmingLanguage)) programmingLanguage else null

Expand Down Expand Up @@ -162,6 +164,9 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se

// When user interacts with a message (e.g. copy code, insert code, vote)
suspend fun recordInteractWithMessage(message: IncomingCwcMessage) {
val hasProjectContext = { messageId: String ->
responseHasProjectContext.getOrDefault(messageId, false)
}
val event: ChatInteractWithMessageEvent? = when (message) {
is IncomingCwcMessage.ChatItemVoted -> {
AmazonqTelemetry.interactWithMessage(
Expand All @@ -173,7 +178,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
else -> CwsprChatInteractionType.Unknown
},
credentialStartUrl = getStartUrl(context.project),
cwsprChatHasProjectContext = getIsProjectContextEnabled()
cwsprChatHasProjectContext = hasProjectContext(message.messageId)
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
Expand All @@ -185,6 +190,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
else -> ChatMessageInteractionType.UNKNOWN_TO_SDK_VERSION
}
)
hasProjectLevelContext(hasProjectContext(message.messageId))
}.build()
}

Expand All @@ -194,12 +200,13 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
cwsprChatMessageId = message.messageId.orEmpty(),
cwsprChatInteractionType = CwsprChatInteractionType.ClickFollowUp,
credentialStartUrl = getStartUrl(context.project),
cwsprChatHasProjectContext = getIsProjectContextEnabled()
cwsprChatHasProjectContext = hasProjectContext(message.messageId.orEmpty())
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
messageId(message.messageId.orEmpty())
interactionType(ChatMessageInteractionType.CLICK_FOLLOW_UP)
hasProjectLevelContext(hasProjectContext(message.messageId.orEmpty()))
}.build()
}

Expand All @@ -214,14 +221,15 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
credentialStartUrl = getStartUrl(context.project),
cwsprChatCodeBlockIndex = message.codeBlockIndex,
cwsprChatTotalCodeBlocks = message.totalCodeBlocks,
cwsprChatHasProjectContext = getIsProjectContextEnabled()
cwsprChatHasProjectContext = hasProjectContext(message.messageId)
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
messageId(message.messageId)
interactionType(ChatMessageInteractionType.COPY_SNIPPET)
interactionTarget(message.insertionTargetType)
acceptedCharacterCount(message.code.length)
hasProjectLevelContext(hasProjectContext(message.messageId))
}.build()
}

Expand All @@ -237,7 +245,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
credentialStartUrl = getStartUrl(context.project),
cwsprChatCodeBlockIndex = message.codeBlockIndex,
cwsprChatTotalCodeBlocks = message.totalCodeBlocks,
cwsprChatHasProjectContext = getIsProjectContextEnabled()
cwsprChatHasProjectContext = hasProjectContext(message.messageId)
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
Expand All @@ -246,6 +254,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
interactionTarget(message.insertionTargetType)
acceptedCharacterCount(message.code.length)
acceptedLineCount(message.code.lines().size)
hasProjectLevelContext(hasProjectContext(message.messageId))
}.build()
}

Expand All @@ -265,7 +274,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
cwsprChatInteractionTarget = message.link,
cwsprChatHasReference = null,
credentialStartUrl = getStartUrl(context.project),
cwsprChatHasProjectContext = getIsProjectContextEnabled()
cwsprChatHasProjectContext = hasProjectContext(message.messageId)
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
Expand All @@ -278,6 +287,7 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
}
)
interactionTarget(message.link)
hasProjectLevelContext(hasProjectContext(message.messageId))
}.build()
}

Expand Down
Loading