Skip to content

Commit 253ccbe

Browse files
leigaolleigaol
authored
telemetry(amazonq): AI code gen % for Q features (#5215)
With the release of many Q features(Inline Suggestion, chat, inline chat, /dev, /test, /doc, /review, /transform), we need to know the % code written by all Q features. This requires calculating and reporting the user written code. The reporting of the code contribution of each Q features was already implemented. % Code Written by Q = Code Written by Q / ( Code Written by Q + Code Written by User) Ref: aws/aws-toolkit-vscode#5991 Calculate and report the user written code for each language by listening to document change events while Q is not making changes to the editor. We add flags to know whether Q is making temporary changes for suggestion rendering or Q suggestion is accepted, by doing so, the document change events are coming from the user. We ignore certain document changes when their length of new characters exceeds 50. Previous data driven research has shown that user tend to copy a huge file from one place to another, making the user written code count skyrocketing but that is actually some existing code not written by the user. We plan to first collect data from IDEs and let it run in the background in shadow mode before we finish the service side aggregation, fix possible bugs and eventually present the AI code written % to the customers.
1 parent f8b15b1 commit 253ccbe

File tree

19 files changed

+445
-9
lines changed

19 files changed

+445
-9
lines changed

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeScan/controller/CodeScanChatHelper.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import software.aws.toolkits.jetbrains.services.amazonqCodeScan.messages.CodeSca
1111
import software.aws.toolkits.jetbrains.services.amazonqCodeScan.messages.CodeScanChatMessageContent
1212
import software.aws.toolkits.jetbrains.services.amazonqCodeScan.messages.UpdatePlaceholderMessage
1313
import software.aws.toolkits.jetbrains.services.amazonqCodeScan.storage.ChatSessionStorage
14+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
15+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
1416
import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
1517
import java.util.UUID
1618

@@ -34,7 +36,7 @@ class CodeScanChatHelper(
3436
clearPreviousItemButtons: Boolean? = false,
3537
) {
3638
if (isInValidSession()) return
37-
39+
broadcastQEvent(QFeatureEvent.INVOCATION)
3840
messagePublisher.publish(
3941
CodeScanChatMessage(
4042
tabId = activeCodeScanTabId as String,

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/controller/CodeTestChatController.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendA
7171
import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
7272
import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
7373
import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage
74+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
75+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
7476
import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
7577
import software.aws.toolkits.jetbrains.services.codewhisperer.util.isWithin
7678
import software.aws.toolkits.jetbrains.services.cwc.ChatConstants
@@ -1205,6 +1207,7 @@ class CodeTestChatController(
12051207
"Processing message: $message " +
12061208
"tabId: $tabId"
12071209
}
1210+
broadcastQEvent(QFeatureEvent.INVOCATION)
12081211
when (session.conversationState) {
12091212
ConversationState.WAITING_FOR_BUILD_COMMAND_INPUT -> handleBuildCommandInput(session, message)
12101213
ConversationState.WAITING_FOR_REGENERATE_INPUT -> handleRegenerateInput(session, message)

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/controller/DocController.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Delete
7070
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.NewFileZipInfo
7171
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.SessionStatePhase
7272
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.CancellationTokenSource
73+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
74+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
7375
import software.aws.toolkits.resources.message
7476
import java.nio.file.Paths
7577
import java.util.UUID
@@ -718,6 +720,7 @@ class DocController(
718720
is PrepareDocGenerationState -> state.filePaths
719721
else -> emptyList()
720722
}
723+
broadcastQEvent(QFeatureEvent.INVOCATION)
721724

722725
if (filePaths.isNotEmpty()) {
723726
processOpenDiff(

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevController.kt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Sessio
7171
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.storage.ChatSessionStorage
7272
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.InsertAction
7373
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.getFollowUpOptions
74+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
75+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
7476
import software.aws.toolkits.jetbrains.services.codewhisperer.util.content
7577
import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.FeedbackComment
7678
import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
@@ -191,6 +193,7 @@ class FeatureDevController(
191193
logger.debug { "$FEATURE_NAME: Processing InsertCodeAtCursorPosition: $message" }
192194

193195
withContext(EDT) {
196+
broadcastQEvent(QFeatureEvent.STARTS_EDITING)
194197
val editor: Editor = FileEditorManager.getInstance(context.project).selectedTextEditor ?: return@withContext
195198

196199
val caret: Caret = editor.caretModel.primaryCaret
@@ -202,6 +205,7 @@ class FeatureDevController(
202205
}
203206
editor.document.insertString(offset, message.code)
204207
}
208+
broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
205209
}
206210
}
207211

@@ -679,7 +683,7 @@ class FeatureDevController(
679683
}
680684

681685
session.preloader(message, messenger)
682-
686+
broadcastQEvent(QFeatureEvent.INVOCATION)
683687
when (session.sessionState.phase) {
684688
SessionStatePhase.CODEGEN -> onCodeGeneration(session, message, tabId)
685689
else -> null

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextCo
4949
import software.aws.toolkits.jetbrains.services.amazonq.project.RelevantDocument
5050
import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhispererConfigurable
5151
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererUserModificationTracker
52+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
53+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
5254
import software.aws.toolkits.jetbrains.services.cwc.InboundAppMessagesHandler
5355
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
5456
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
@@ -215,6 +217,7 @@ class ChatController private constructor(
215217
}
216218

217219
override suspend fun processInsertCodeAtCursorPosition(message: IncomingCwcMessage.InsertCodeAtCursorPosition) {
220+
broadcastQEvent(QFeatureEvent.STARTS_EDITING)
218221
withContext(EDT) {
219222
val editor: Editor = FileEditorManager.getInstance(context.project).selectedTextEditor ?: return@withContext
220223

@@ -245,6 +248,8 @@ class ChatController private constructor(
245248
}
246249
}
247250
telemetryHelper.recordInteractWithMessage(message)
251+
252+
broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
248253
}
249254

250255
override suspend fun processStopResponseMessage(message: IncomingCwcMessage.StopResponse) {
@@ -438,7 +443,7 @@ class ChatController private constructor(
438443
sessionInfo.history.add(requestData)
439444
telemetryHelper.recordEnterFocusConversation(tabId)
440445
telemetryHelper.recordStartConversation(tabId, requestData)
441-
446+
broadcastQEvent(QFeatureEvent.INVOCATION)
442447
// Send the request to the API and publish the responses back to the UI.
443448
// This is launched in a scope attached to the sessionInfo so that the Job can be cancelled on a per-session basis.
444449
ChatPromptHandler(telemetryHelper).handle(tabId, triggerId, requestData, sessionInfo, shouldAddIndexInProgressMessage)

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException
1313
import software.amazon.awssdk.services.codewhispererstreaming.model.CodeWhispererStreamingException
1414
import software.aws.toolkits.core.utils.convertMarkdownToHTML
1515
import software.aws.toolkits.core.utils.extractCodeBlockLanguage
16+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
17+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
1618
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
1719
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
1820
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatResponseEvent
@@ -115,6 +117,8 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
115117
)
116118
telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode, countTotalNumberOfCodeBlocks(responseText))
117119
emit(response)
120+
121+
broadcastQEvent(QFeatureEvent.INVOCATION)
118122
}
119123
.catch { exception ->
120124
val statusCode = if (exception is AwsServiceException) exception.statusCode() else 0

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/inline/InlineChatController.kt‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
5858
import software.aws.toolkits.jetbrains.services.amazonq.toolwindow.AMAZON_Q_WINDOW_ID
5959
import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator
6060
import software.aws.toolkits.jetbrains.services.codewhisperer.model.CaretPosition
61+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
62+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
6163
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
6264
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.TriggerType
6365
import software.aws.toolkits.jetbrains.services.cwc.controller.ReferenceLogController
@@ -191,7 +193,6 @@ class InlineChatController(
191193

192194
private fun addPopupListeners(popup: JBPopup, editor: Editor) {
193195
val popupListener = object : JBPopupListener {
194-
195196
override fun onClosed(event: LightweightWindowEvent) {
196197
if (canPopupAbort.get() && event.asPopup().isDisposed) {
197198
popupCancelHandler.invoke(editor)
@@ -534,6 +535,7 @@ class InlineChatController(
534535
private fun insertString(editor: Editor, offset: Int, text: String): RangeMarker {
535536
lateinit var rangeMarker: RangeMarker
536537

538+
broadcastQEvent(QFeatureEvent.STARTS_EDITING)
537539
ApplicationManager.getApplication().invokeAndWait {
538540
CommandProcessor.getInstance().runUndoTransparentAction {
539541
WriteCommandAction.runWriteCommandAction(project) {
@@ -543,18 +545,20 @@ class InlineChatController(
543545
highlightCodeWithBackgroundColor(editor, rangeMarker.startOffset, rangeMarker.endOffset, true)
544546
}
545547
}
546-
548+
broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
547549
return rangeMarker
548550
}
549551

550552
private fun replaceString(document: Document, start: Int, end: Int, text: String) {
553+
broadcastQEvent(QFeatureEvent.STARTS_EDITING)
551554
ApplicationManager.getApplication().invokeAndWait {
552555
CommandProcessor.getInstance().runUndoTransparentAction {
553556
WriteCommandAction.runWriteCommandAction(project) {
554557
document.replaceString(start, end, text)
555558
}
556559
}
557560
}
561+
broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
558562
}
559563

560564
private fun highlightString(editor: Editor, start: Int, end: Int, isInsert: Boolean) {
@@ -711,6 +715,8 @@ class InlineChatController(
711715
canPopupAbort.set(true)
712716
undoChanges()
713717
}
718+
719+
broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
714720
return errorMessage
715721
}
716722

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/controller/CodeTransformChatController.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.utils.toVirtualFi
111111
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.tryGetJdk
112112
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.unzipFile
113113
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.validateSctMetadata
114+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
115+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
114116
import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
115117
import software.aws.toolkits.resources.message
116118

@@ -136,7 +138,7 @@ class CodeTransformChatController(
136138
if (objective == "language upgrade" || objective == "sql conversion") {
137139
telemetry.submitSelection(objective)
138140
}
139-
141+
broadcastQEvent(QFeatureEvent.INVOCATION)
140142
when (objective) {
141143
"language upgrade" -> this.handleLanguageUpgrade()
142144
"sql conversion" -> this.handleSQLConversion()

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/utils/CodeWhispererCodeScanIssueUtils.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhisp
3838
import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
3939
import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage
4040
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
41+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
42+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
4143
import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
4244
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
4345
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.CODE_SCAN_ISSUE_TITLE_MAX_LENGTH
@@ -331,6 +333,7 @@ fun applySuggestedFix(project: Project, issue: CodeWhispererCodeScanIssue) {
331333
try {
332334
val manager = CodeWhispererCodeReferenceManager.getInstance(issue.project)
333335
WriteCommandAction.runWriteCommandAction(issue.project) {
336+
broadcastQEvent(QFeatureEvent.STARTS_EDITING)
334337
val document = FileDocumentManager.getInstance().getDocument(issue.file) ?: return@runWriteCommandAction
335338

336339
val documentContent = document.text
@@ -343,6 +346,7 @@ fun applySuggestedFix(project: Project, issue: CodeWhispererCodeScanIssue) {
343346
LOG.debug { "Original content from reference span: $originalContent" }
344347
manager.addReferenceLogPanelEntry(reference = reference, null, null, originalContent.split("\n"))
345348
}
349+
broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
346350
}
347351
if (issue.suggestedFixes[0].references.isNotEmpty()) {
348352
manager.toolWindow?.show()

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ interface CodeWhispererClientAdaptor : Disposable {
134134
acceptedTokenCount: Long,
135135
totalTokenCount: Long,
136136
unmodifiedAcceptedTokenCount: Long?,
137+
userWrittenCodeCharacterCount: Long?,
138+
userWrittenCodeLineCount: Long?,
137139
): SendTelemetryEventResponse
138140

139141
fun sendUserModificationTelemetry(
@@ -481,6 +483,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
481483
acceptedTokenCount: Long,
482484
totalTokenCount: Long,
483485
unmodifiedAcceptedTokenCount: Long?,
486+
userWrittenCodeCharacterCount: Long?,
487+
userWrittenCodeLineCount: Long?,
484488
): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
485489
requestBuilder.telemetryEvent { telemetryEventBuilder ->
486490
telemetryEventBuilder.codeCoverageEvent {
@@ -490,6 +494,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
490494
it.totalCharacterCount(totalTokenCount.toInt())
491495
it.timestamp(Instant.now())
492496
it.unmodifiedAcceptedCharacterCount(unmodifiedAcceptedTokenCount?.toInt())
497+
it.userWrittenCodeCharacterCount(userWrittenCodeLineCount?.toInt())
498+
it.userWrittenCodeLineCount(userWrittenCodeLineCount?.toInt())
493499
}
494500
}
495501
requestBuilder.optOutPreference(getTelemetryOptOutPreference())

0 commit comments

Comments
 (0)