Skip to content

Commit 8b5738e

Browse files
andrewyuqandrewyuq
authored
Enable SendTelemetryEvent for all SSO users (#3835)
SendTelemetryEvent API gets called for 3 payloads: UserTriggerDecision, CodePercentage and CodeScan.
1 parent 31cd1c9 commit 8b5738e

File tree

12 files changed

+684
-68
lines changed

12 files changed

+684
-68
lines changed

‎jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererCodeScanManager.kt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import kotlinx.coroutines.time.withTimeout
4242
import kotlinx.coroutines.withContext
4343
import org.jetbrains.annotations.TestOnly
4444
import software.amazon.awssdk.services.codewhisperer.model.CodeWhispererException
45+
import software.amazon.awssdk.services.codewhispererruntime.model.CodeWhispererRuntimeException
4546
import software.amazon.awssdk.services.codewhispererruntime.model.ThrottlingException
4647
import software.aws.toolkits.core.utils.WaiterTimeoutException
4748
import software.aws.toolkits.core.utils.debug
@@ -56,16 +57,21 @@ import software.aws.toolkits.jetbrains.core.explorer.refreshDevToolTree
5657
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.listeners.CodeWhispererCodeScanDocumentListener
5758
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.listeners.CodeWhispererCodeScanEditorMouseMotionListener
5859
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.CodeScanSessionConfig
60+
import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
5961
import software.aws.toolkits.jetbrains.services.codewhisperer.editor.CodeWhispererEditorUtil.overlaps
6062
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhispererExplorerActionManager
6163
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.isCodeWhispererEnabled
64+
import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
65+
import software.aws.toolkits.jetbrains.services.codewhisperer.language.languages.CodeWhispererUnknownLanguage
66+
import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage
6267
import software.aws.toolkits.jetbrains.services.codewhisperer.model.CodeScanTelemetryEvent
6368
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
6469
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererColorUtil.INACTIVE_TEXT_COLOR
6570
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
6671
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.ISSUE_HIGHLIGHT_TEXT_ATTRIBUTES
6772
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil
6873
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.promptReAuth
74+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.runIfIamIdentityCenterConnection
6975
import software.aws.toolkits.resources.message
7076
import software.aws.toolkits.telemetry.Result
7177
import java.time.Duration
@@ -138,6 +144,7 @@ class CodeWhispererCodeScanManager(val project: Project) {
138144

139145
// Prepare for a code scan
140146
beforeCodeScan()
147+
141148
// launch code scan coroutine
142149
codeScanJob = launchCodeScanCoroutine()
143150
}
@@ -166,17 +173,21 @@ class CodeWhispererCodeScanManager(val project: Project) {
166173
var codeScanResponseContext = defaultCodeScanResponseContext()
167174
var getProjectSize: Deferred<Long?> = async { null }
168175
val connection = ToolkitConnectionManager.getInstance(project).activeConnectionForFeature(CodeWhispererConnection.getInstance())
176+
var codeScanJobId: String? = null
177+
var language: CodeWhispererProgrammingLanguage = CodeWhispererUnknownLanguage.INSTANCE
169178
try {
170179
val file = FileEditorManager.getInstance(project).selectedEditor?.file
171180
?: noFileOpenError()
172181
val codeScanSessionConfig = CodeScanSessionConfig.create(file, project)
182+
language = codeScanSessionConfig.getSelectedFile().programmingLanguage()
173183
withTimeout(Duration.ofSeconds(codeScanSessionConfig.overallJobTimeoutInSeconds())) {
174184
// 1. Generate truncation (zip files) based on the current editor.
175185
LOG.debug { "Creating context truncation for file ${file.path}" }
176186
val sessionContext = CodeScanSessionContext(project, codeScanSessionConfig)
177187
val session = CodeWhispererCodeScanSession(sessionContext)
178188
val codeScanResponse = session.run()
179189
codeScanResponseContext = codeScanResponse.responseContext
190+
codeScanJobId = codeScanResponseContext.codeScanJobId
180191
when (codeScanResponse) {
181192
is CodeScanResponse.Success -> {
182193
val issues = codeScanResponse.issues
@@ -213,6 +224,7 @@ class CodeWhispererCodeScanManager(val project: Project) {
213224
CodeWhispererTelemetryService.getInstance().sendSecurityScanEvent(
214225
CodeScanTelemetryEvent(codeScanResponseContext, duration, codeScanStatus, getProjectSize.await()?.toDouble(), connection)
215226
)
227+
sendCodeScanTelemetryToServiceAPI(project, language, codeScanJobId)
216228
}
217229
}
218230
}
@@ -387,6 +399,25 @@ class CodeWhispererCodeScanManager(val project: Project) {
387399
project.refreshDevToolTree()
388400
}
389401

402+
private fun sendCodeScanTelemetryToServiceAPI(
403+
project: Project,
404+
programmingLanguage: CodeWhispererProgrammingLanguage,
405+
codeScanJobId: String?
406+
) {
407+
runIfIamIdentityCenterConnection(project) {
408+
try {
409+
val response = CodeWhispererClientAdaptor.getInstance(project)
410+
.sendCodeScanTelemetry(programmingLanguage, codeScanJobId)
411+
LOG.debug { "Successfully sent code scan telemetry. RequestId: ${response.responseMetadata().requestId()}" }
412+
} catch (e: Exception) {
413+
val requestId = if (e is CodeWhispererRuntimeException) e.requestId() else null
414+
LOG.debug {
415+
"Failed to send code scan telemetry. RequestId: $requestId, ErrorMessage: ${e.message}"
416+
}
417+
}
418+
}
419+
}
420+
390421
/**
391422
* Creates a CodeWhisperer code scan issues tree.
392423
* For each scan node:

‎jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/sessionconfig/CodeScanSessionConfig.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ sealed class CodeScanSessionConfig(
5555

5656
open fun getImportedFiles(file: VirtualFile, includedSourceFiles: Set<String>): List<String> = listOf()
5757

58+
open fun getSelectedFile(): VirtualFile = selectedFile
59+
5860
open fun createPayload(): Payload {
5961
// Fail fast if the selected file size is greater than the payload limit.
6062
if (selectedFile.length > getPayloadLimitInBytes()) {

‎jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@ import software.amazon.awssdk.services.codewhisperer.model.GetCodeScanResponse
1616
import software.amazon.awssdk.services.codewhisperer.model.ListCodeScanFindingsRequest
1717
import software.amazon.awssdk.services.codewhisperer.model.ListCodeScanFindingsResponse
1818
import software.amazon.awssdk.services.codewhispererruntime.CodeWhispererRuntimeClient
19+
import software.amazon.awssdk.services.codewhispererruntime.model.CompletionType
1920
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlRequest
2021
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
2122
import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsRequest
2223
import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsResponse
24+
import software.amazon.awssdk.services.codewhispererruntime.model.SendTelemetryEventResponse
25+
import software.amazon.awssdk.services.codewhispererruntime.model.SuggestionState
2326
import software.aws.toolkits.core.utils.getLogger
2427
import software.aws.toolkits.core.utils.warn
2528
import software.aws.toolkits.jetbrains.core.AwsClientManager
@@ -31,8 +34,15 @@ import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnectionManager
3134
import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnectionManagerListener
3235
import software.aws.toolkits.jetbrains.core.credentials.pinning.CodeWhispererConnection
3336
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhispererExplorerActionManager
37+
import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
38+
import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestContext
39+
import software.aws.toolkits.jetbrains.services.codewhisperer.service.ResponseContext
3440
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
3541
import software.aws.toolkits.jetbrains.services.codewhisperer.util.transform
42+
import software.aws.toolkits.telemetry.CodewhispererCompletionType
43+
import software.aws.toolkits.telemetry.CodewhispererSuggestionState
44+
import java.time.Instant
45+
import java.util.concurrent.TimeUnit
3646
import kotlin.reflect.KProperty0
3747
import kotlin.reflect.jvm.isAccessible
3848

@@ -64,11 +74,40 @@ interface CodeWhispererClientAdaptor : Disposable {
6474
isSigv4: Boolean = shouldUseSigv4Client(project)
6575
): ListCodeScanFindingsResponse
6676

77+
fun putUserTriggerDecisionTelemetry(
78+
requestContext: RequestContext,
79+
responseContext: ResponseContext,
80+
completionType: CodewhispererCompletionType,
81+
suggestionState: CodewhispererSuggestionState,
82+
suggestionReferenceCount: Int,
83+
lineCount: Int
84+
): SendTelemetryEventResponse
85+
86+
fun putCodePercentageTelemetry(
87+
language: CodeWhispererProgrammingLanguage,
88+
acceptedTokenCount: Int,
89+
totalTokenCount: Int
90+
): SendTelemetryEventResponse
91+
92+
fun sendUserModificationTelemetry(
93+
sessionId: String,
94+
requestId: String,
95+
language: CodeWhispererProgrammingLanguage,
96+
modificationPercentage: Double
97+
): SendTelemetryEventResponse
98+
99+
fun sendCodeScanTelemetry(
100+
language: CodeWhispererProgrammingLanguage,
101+
codeScanJobId: String?
102+
): SendTelemetryEventResponse
103+
67104
companion object {
68105
fun getInstance(project: Project): CodeWhispererClientAdaptor = project.service()
69106

70107
private fun shouldUseSigv4Client(project: Project) =
71108
CodeWhispererExplorerActionManager.getInstance().checkActiveCodeWhispererConnectionType(project) == CodeWhispererLoginType.Accountless
109+
110+
const val INVALID_CODESCANJOBID = "Invalid_CodeScanJobID"
72111
}
73112
}
74113

@@ -139,6 +178,91 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
139178
bearerClient().listCodeAnalysisFindings(request.transform()).transform()
140179
}
141180

181+
override fun putUserTriggerDecisionTelemetry(
182+
requestContext: RequestContext,
183+
responseContext: ResponseContext,
184+
completionType: CodewhispererCompletionType,
185+
suggestionState: CodewhispererSuggestionState,
186+
suggestionReferenceCount: Int,
187+
lineCount: Int
188+
): SendTelemetryEventResponse {
189+
val fileContext = requestContext.fileContextInfo
190+
val programmingLanguage = fileContext.programmingLanguage
191+
var e2eLatency = requestContext.latencyContext.getCodeWhispererEndToEndLatency()
192+
193+
// When we send a userTriggerDecision of Empty or Discard, we set the time users see the first
194+
// suggestion to be now.
195+
if (e2eLatency < 0) {
196+
e2eLatency = TimeUnit.NANOSECONDS.toMillis(
197+
System.nanoTime() - requestContext.latencyContext.codewhispererEndToEndStart
198+
).toDouble()
199+
}
200+
return bearerClient().sendTelemetryEvent { requestBuilder ->
201+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
202+
telemetryEventBuilder.userTriggerDecisionEvent {
203+
it.requestId(requestContext.latencyContext.firstRequestId)
204+
it.completionType(completionType.toCodeWhispererSdkType())
205+
it.programmingLanguage { builder -> builder.languageName(programmingLanguage.languageId) }
206+
it.sessionId(responseContext.sessionId)
207+
it.recommendationLatencyMilliseconds(e2eLatency)
208+
it.suggestionState(suggestionState.toCodeWhispererSdkType())
209+
it.timestamp(Instant.now())
210+
it.suggestionReferenceCount(suggestionReferenceCount)
211+
it.generatedLine(lineCount)
212+
}
213+
}
214+
}
215+
}
216+
217+
override fun putCodePercentageTelemetry(
218+
language: CodeWhispererProgrammingLanguage,
219+
acceptedTokenCount: Int,
220+
totalTokenCount: Int
221+
): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
222+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
223+
telemetryEventBuilder.codeCoverageEvent {
224+
it.programmingLanguage { languageBuilder -> languageBuilder.languageName(language.languageId) }
225+
it.acceptedCharacterCount(acceptedTokenCount)
226+
it.totalCharacterCount(totalTokenCount)
227+
it.timestamp(Instant.now())
228+
}
229+
}
230+
}
231+
232+
override fun sendUserModificationTelemetry(
233+
sessionId: String,
234+
requestId: String,
235+
language: CodeWhispererProgrammingLanguage,
236+
modificationPercentage: Double
237+
): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
238+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
239+
telemetryEventBuilder.userModificationEvent {
240+
it.sessionId(sessionId)
241+
it.requestId(requestId)
242+
it.programmingLanguage { languageBuilder ->
243+
languageBuilder.languageName(language.languageId)
244+
}
245+
it.modificationPercentage(modificationPercentage)
246+
it.timestamp(Instant.now())
247+
}
248+
}
249+
}
250+
251+
override fun sendCodeScanTelemetry(
252+
language: CodeWhispererProgrammingLanguage,
253+
codeScanJobId: String?
254+
): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
255+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
256+
telemetryEventBuilder.codeScanEvent {
257+
it.programmingLanguage { languageBuilder ->
258+
languageBuilder.languageName(language.languageId)
259+
}
260+
it.codeScanJobId(if (codeScanJobId.isNullOrEmpty()) CodeWhispererClientAdaptor.INVALID_CODESCANJOBID else codeScanJobId)
261+
it.timestamp(Instant.now())
262+
}
263+
}
264+
}
265+
142266
override fun dispose() {
143267
if (this::mySigv4Client.isLazyInitialized) {
144268
mySigv4Client.close()
@@ -184,3 +308,17 @@ class MockCodeWhispererClientAdaptor(override val project: Project) : CodeWhispe
184308

185309
override fun dispose() {}
186310
}
311+
312+
private fun CodewhispererSuggestionState.toCodeWhispererSdkType() = when {
313+
this == CodewhispererSuggestionState.Accept -> SuggestionState.ACCEPT
314+
this == CodewhispererSuggestionState.Reject -> SuggestionState.REJECT
315+
this == CodewhispererSuggestionState.Empty -> SuggestionState.EMPTY
316+
this == CodewhispererSuggestionState.Discard -> SuggestionState.DISCARD
317+
else -> SuggestionState.UNKNOWN_TO_SDK_VERSION
318+
}
319+
320+
private fun CodewhispererCompletionType.toCodeWhispererSdkType() = when {
321+
this == CodewhispererCompletionType.Line -> CompletionType.LINE
322+
this == CodewhispererCompletionType.Block -> CompletionType.BLOCK
323+
else -> CompletionType.UNKNOWN_TO_SDK_VERSION
324+
}

‎jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/editor/CodeWhispererEditorListener.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ class CodeWhispererEditorListener : EditorFactoryListener {
2727
override fun documentChanged(event: DocumentEvent) {
2828
if (!isCodeWhispererEnabled(project)) return
2929
CodeWhispererInvocationStatus.getInstance().documentChanged()
30-
CodeWhispererCodeCoverageTracker.getInstance(language).apply {
30+
CodeWhispererCodeCoverageTracker.getInstance(project, language).apply {
3131
activateTrackerIfNotActive()
3232
documentChanged(event)
3333
}

0 commit comments

Comments
 (0)