Skip to content

Commit dd01382

Browse files
CodeWhisperer: ab testing feature (#4020)
* Add optional optout field to GenerateCompletion API for telemetry optout (#3964) * CodeWhisperer: Add feature fetching component (#3978) * CodeWhisperer: Add feature fetching component 1. Add a component to fetch feature assignments every 30 mins, calling ListFeatureEvaluations API and cache values in memory. 2. Add a field in UserTriggerDecision codewhispererFeatureEvaluations to record all the feature configs in a string, which can be queried on Kibana * fix detektMain * leftover changes * detekt fix * Add clientId and numberOfRecommendations field (#4017) * Add clientId and numberOfRecommendations field * Add userContext and revert the latency change * fix tests --------- Co-authored-by: aws-toolkit-automation <[email protected]>
1 parent 18e783f commit dd01382

File tree

15 files changed

+277
-29
lines changed

15 files changed

+277
-29
lines changed

gradle/libs.versions.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ kotlinCoroutines = "1.6.4"
2020
mockito = "4.6.1"
2121
mockitoKotlin = "4.0.0"
2222
mockk = "1.13.8"
23-
telemetryGenerator = "1.0.161"
23+
telemetryGenerator = "1.0.162"
2424
testLogger = "3.1.0"
2525
testRetry = "1.5.2"
2626
slf4j = "1.7.36"

jetbrains-core/resources/META-INF/plugin.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,7 @@
355355
<applicationService serviceImplementation="software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererUserGroupSettings"/>
356356
<applicationService serviceInterface="software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator"
357357
serviceImplementation="software.aws.toolkits.jetbrains.services.codewhisperer.customization.DefaultCodeWhispererModelConfigurator"/>
358+
<applicationService serviceImplementation="software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererFeatureConfigService"/>
358359

359360
<projectService serviceImplementation="software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager"/>
360361
<projectService serviceInterface="software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor"

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.Dimension
2323
import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsRequest
2424
import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsResponse
2525
import software.amazon.awssdk.services.codewhispererruntime.model.ListAvailableCustomizationsRequest
26-
import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
26+
import software.amazon.awssdk.services.codewhispererruntime.model.ListFeatureEvaluationsResponse
2727
import software.amazon.awssdk.services.codewhispererruntime.model.SendTelemetryEventResponse
2828
import software.amazon.awssdk.services.codewhispererruntime.model.SuggestionState
2929
import software.aws.toolkits.core.utils.debug
@@ -42,8 +42,9 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhisp
4242
import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestContext
4343
import software.aws.toolkits.jetbrains.services.codewhisperer.service.ResponseContext
4444
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
45+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
4546
import software.aws.toolkits.jetbrains.services.codewhisperer.util.transform
46-
import software.aws.toolkits.jetbrains.settings.AwsSettings
47+
import software.aws.toolkits.jetbrains.services.telemetry.ClientMetadata
4748
import software.aws.toolkits.telemetry.CodewhispererCompletionType
4849
import software.aws.toolkits.telemetry.CodewhispererSuggestionState
4950
import java.time.Instant
@@ -88,7 +89,8 @@ interface CodeWhispererClientAdaptor : Disposable {
8889
completionType: CodewhispererCompletionType,
8990
suggestionState: CodewhispererSuggestionState,
9091
suggestionReferenceCount: Int,
91-
lineCount: Int
92+
lineCount: Int,
93+
numberOfRecommendations: Int
9294
): SendTelemetryEventResponse
9395

9496
fun sendCodePercentageTelemetry(
@@ -111,6 +113,8 @@ interface CodeWhispererClientAdaptor : Disposable {
111113
codeScanJobId: String?
112114
): SendTelemetryEventResponse
113115

116+
fun listFeatureEvaluations(): ListFeatureEvaluationsResponse
117+
114118
fun sendMetricDataTelemetry(eventName: String, metadata: Map<String, Any?>): SendTelemetryEventResponse
115119

116120
companion object {
@@ -218,7 +222,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
218222
completionType: CodewhispererCompletionType,
219223
suggestionState: CodewhispererSuggestionState,
220224
suggestionReferenceCount: Int,
221-
lineCount: Int
225+
lineCount: Int,
226+
numberOfRecommendations: Int
222227
): SendTelemetryEventResponse {
223228
val fileContext = requestContext.fileContextInfo
224229
val programmingLanguage = fileContext.programmingLanguage
@@ -244,9 +249,11 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
244249
it.suggestionReferenceCount(suggestionReferenceCount)
245250
it.generatedLine(lineCount)
246251
it.customizationArn(requestContext.customizationArn)
252+
it.numberOfRecommendations(numberOfRecommendations)
247253
}
248254
}
249-
requestBuilder.optOutPreference(getTelemetryOptoutPreference())
255+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
256+
requestBuilder.userContext(ClientMetadata.DEFAULT_METADATA.codeWhispererUserContext)
250257
}
251258
}
252259

@@ -265,7 +272,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
265272
it.timestamp(Instant.now())
266273
}
267274
}
268-
requestBuilder.optOutPreference(getTelemetryOptoutPreference())
275+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
276+
requestBuilder.userContext(ClientMetadata.DEFAULT_METADATA.codeWhispererUserContext)
269277
}
270278

271279
override fun sendUserModificationTelemetry(
@@ -287,7 +295,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
287295
it.timestamp(Instant.now())
288296
}
289297
}
290-
requestBuilder.optOutPreference(getTelemetryOptoutPreference())
298+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
299+
requestBuilder.userContext(ClientMetadata.DEFAULT_METADATA.codeWhispererUserContext)
291300
}
292301

293302
override fun sendCodeScanTelemetry(
@@ -303,7 +312,12 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
303312
it.timestamp(Instant.now())
304313
}
305314
}
306-
requestBuilder.optOutPreference(getTelemetryOptoutPreference())
315+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
316+
requestBuilder.userContext(ClientMetadata.DEFAULT_METADATA.codeWhispererUserContext)
317+
}
318+
319+
override fun listFeatureEvaluations(): ListFeatureEvaluationsResponse = bearerClient().listFeatureEvaluations {
320+
it.userContext(ClientMetadata.DEFAULT_METADATA.codeWhispererUserContext)
307321
}
308322

309323
override fun sendMetricDataTelemetry(eventName: String, metadata: Map<String, Any?>): SendTelemetryEventResponse =
@@ -315,15 +329,9 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
315329
metricBuilder.timestamp(Instant.now())
316330
metricBuilder.dimensions(metadata.filter { it.value != null }.map { Dimension.builder().name(it.key).value(it.value.toString()).build() })
317331
}
318-
requestBuilder.optOutPreference(getTelemetryOptoutPreference())
319332
}
320-
}
321-
322-
private fun getTelemetryOptoutPreference() =
323-
if (AwsSettings.getInstance().isTelemetryEnabled) {
324-
OptOutPreference.OPTIN
325-
} else {
326-
OptOutPreference.OPTOUT
333+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
334+
requestBuilder.userContext(ClientMetadata.DEFAULT_METADATA.codeWhispererUserContext)
327335
}
328336

329337
override fun dispose() {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package software.aws.toolkits.jetbrains.services.codewhisperer.service
5+
6+
import com.intellij.openapi.components.service
7+
import com.intellij.openapi.project.Project
8+
import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
9+
import software.amazon.awssdk.services.codewhispererruntime.model.FeatureValue
10+
import software.aws.toolkits.core.utils.debug
11+
import software.aws.toolkits.core.utils.getLogger
12+
import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
13+
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.isCodeWhispererExpired
14+
15+
class CodeWhispererFeatureConfigService {
16+
private val featureConfigs = mutableMapOf<String, FeatureContext>()
17+
18+
@RequiresBackgroundThread
19+
fun fetchFeatureConfigs(project: Project) {
20+
if (isCodeWhispererExpired(project)) return
21+
22+
LOG.debug { "Fetching feature configs" }
23+
try {
24+
val response = CodeWhispererClientAdaptor.getInstance(project).listFeatureEvaluations()
25+
26+
// Simply force overwrite feature configs from server response, no needed to check existing values.
27+
response.featureEvaluations().forEach {
28+
featureConfigs[it.feature()] = FeatureContext(it.feature(), it.variation(), it.value())
29+
}
30+
} catch (e: Exception) {
31+
LOG.debug(e) { "Error when fetching feature configs" }
32+
}
33+
LOG.debug { "Current feature configs: ${getFeatureConfigsTelemetry()}" }
34+
}
35+
36+
fun getFeatureConfigsTelemetry(): String =
37+
"{${featureConfigs.entries.joinToString(", ") { (name, context) ->
38+
"$name: ${context.variation}"
39+
}}}"
40+
41+
// TODO: for all feature variations, define a contract that can be enforced upon the implementation of
42+
// the business logic.
43+
// When we align on a new feature config, client-side will implement specific business logic to utilize
44+
// these values by:
45+
// 1) Add an entry in FEATURE_DEFINITIONS, which is <feature_name> to <feature_context>.
46+
// 2) Add a function with name `getXXX`, where XXX refers to the feature name.
47+
// 3) Specify the return type: One of the return type String/Boolean/Long/Double should be used here.
48+
// 4) Specify the key for the `getFeatureValueForKey` helper function which is the feature name.
49+
// 5) Specify the corresponding type value getter for the `FeatureValue` class. For example,
50+
// if the return type is Long, then the corresponding type value getter is `longValue()`.
51+
// 6) Add a test case for this feature.
52+
fun getTestFeature(): String = getFeatureValueForKey(TEST_FEATURE_NAME).stringValue()
53+
54+
// Get the feature value for the given key.
55+
// In case of a misconfiguration, it will return a default feature value of Boolean true.
56+
private fun getFeatureValueForKey(name: String): FeatureValue =
57+
featureConfigs[name]?.value ?: FEATURE_DEFINITIONS[name]?.value
58+
?: FeatureValue.builder().boolValue(true).build()
59+
60+
companion object {
61+
fun getInstance(): CodeWhispererFeatureConfigService = service()
62+
private const val TEST_FEATURE_NAME = "testFeature"
63+
private val LOG = getLogger<CodeWhispererFeatureConfigService>()
64+
65+
// TODO: add real feature later
66+
internal val FEATURE_DEFINITIONS = mapOf(
67+
TEST_FEATURE_NAME to FeatureContext(
68+
TEST_FEATURE_NAME,
69+
"CONTROL",
70+
FeatureValue.builder().stringValue("testValue").build()
71+
)
72+
)
73+
}
74+
}
75+
76+
data class FeatureContext(
77+
val name: String,
78+
val variation: String,
79+
val value: FeatureValue
80+
)

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.util.CaretMovement
7373
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
7474
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.SUPPLEMENTAL_CONTEXT_TIMEOUT
7575
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCompletionType
76+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
7677
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.notifyErrorCodeWhispererUsageLimit
7778
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.promptReAuth
7879
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CrossFileStrategy
@@ -778,6 +779,7 @@ class CodeWhispererService {
778779
.supplementalContexts(supplementalContexts)
779780
.referenceTrackerConfiguration { it.recommendationsWithReferences(includeCodeWithReference) }
780781
.customizationArn(customizationArn)
782+
.optOutPreference(getTelemetryOptOutPreference())
781783
.build()
782784
}
783785
}

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/startup/CodeWhispererProjectStartupActivity.kt

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,21 @@ import com.intellij.openapi.application.ApplicationManager
88
import com.intellij.openapi.application.invokeLater
99
import com.intellij.openapi.project.Project
1010
import com.intellij.openapi.startup.StartupActivity
11+
import kotlinx.coroutines.delay
12+
import kotlinx.coroutines.isActive
13+
import kotlinx.coroutines.launch
14+
import software.aws.toolkits.jetbrains.core.coroutines.projectCoroutineScope
1115
import software.aws.toolkits.jetbrains.core.explorer.refreshCwQTree
1216
import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererLoginType
1317
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhispererExplorerActionManager
1418
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.isCodeWhispererEnabled
19+
import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.isCodeWhispererExpired
1520
import software.aws.toolkits.jetbrains.services.codewhisperer.importadder.CodeWhispererImportAdderListener
1621
import software.aws.toolkits.jetbrains.services.codewhisperer.popup.CodeWhispererPopupManager.Companion.CODEWHISPERER_USER_ACTION_PERFORMED
22+
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererFeatureConfigService
1723
import software.aws.toolkits.jetbrains.services.codewhisperer.status.CodeWhispererStatusBarManager
1824
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
25+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.FEATURE_CONFIG_POLL_INTERVAL_IN_MS
1926
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.notifyErrorAccountless
2027
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.notifyWarnAccountless
2128
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.promptReAuth
@@ -42,8 +49,12 @@ class CodeWhispererProjectStartupActivity : StartupActivity.DumbAware {
4249

4350
// Reconnect CodeWhisperer on startup
4451
promptReAuth(project, isPluginStarting = true)
52+
if (isCodeWhispererExpired(project)) return
4553

46-
// install intellsense autotrigger listener, this only need to be executed 1 time
54+
// Init featureConfig job
55+
initFeatureConfigPollingJob(project)
56+
57+
// install intellsense autotrigger listener, this only need to be executed once
4758
project.messageBus.connect().subscribe(LookupManagerListener.TOPIC, CodeWhispererIntelliSenseAutoTriggerListener)
4859
project.messageBus.connect().subscribe(CODEWHISPERER_USER_ACTION_PERFORMED, CodeWhispererImportAdderListener)
4960

@@ -104,6 +115,16 @@ class CodeWhispererProjectStartupActivity : StartupActivity.DumbAware {
104115
parsedLastShown.plusDays(7) <= LocalDateTime.now()
105116
} ?: true
106117
}
118+
119+
// Start a job that runs every 30 mins
120+
private fun initFeatureConfigPollingJob(project: Project) {
121+
projectCoroutineScope(project).launch {
122+
while (isActive) {
123+
CodeWhispererFeatureConfigService.getInstance().fetchFeatureConfigs(project)
124+
delay(FEATURE_CONFIG_POLL_INTERVAL_IN_MS)
125+
}
126+
}
127+
}
107128
}
108129

109130
// TODO: do we have time zone issue with Date?

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererTelemetryService.kt

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.model.Recommendati
2626
import software.aws.toolkits.jetbrains.services.codewhisperer.model.SessionContext
2727
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererAutoTriggerService
2828
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererAutomatedTriggerType
29+
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererFeatureConfigService
2930
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererInvocationStatus
3031
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererUserGroupSettings
3132
import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestContext
@@ -225,7 +226,8 @@ class CodeWhispererTelemetryService {
225226
completionType,
226227
suggestionState,
227228
suggestionReferenceCount,
228-
generatedLineCount
229+
generatedLineCount,
230+
recommendationContext.details.size
229231
)
230232
LOG.debug {
231233
"Successfully sent user trigger decision telemetry. RequestId: ${response.responseMetadata().requestId()}"
@@ -272,7 +274,8 @@ class CodeWhispererTelemetryService {
272274
codewhispererSupplementalContextTimeout = supplementalContext?.isProcessTimeout,
273275
codewhispererSupplementalContextStrategyId = supplementalContext?.strategy.toString(),
274276
codewhispererUserGroup = CodeWhispererUserGroupSettings.getInstance().getUserGroup().name,
275-
codewhispererGettingStartedTask = getGettingStartedTaskType(requestContext.editor)
277+
codewhispererGettingStartedTask = getGettingStartedTaskType(requestContext.editor),
278+
codewhispererFeatureEvaluations = CodeWhispererFeatureConfigService.getInstance().getFeatureConfigsTelemetry()
276279
)
277280
}
278281

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererConstants.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ object CodeWhispererConstants {
3131
const val POPUP_DELAY_CHECK_INTERVAL: Long = 25
3232
const val IDLE_TIME_CHECK_INTERVAL: Long = 25
3333
const val SUPPLEMENTAL_CONTEXT_TIMEOUT = 50L
34+
const val FEATURE_EVALUATION_PRODUCT_NAME = "CodeWhisperer"
3435

3536
val AWSTemplateKeyWordsRegex = Regex("(AWSTemplateFormatVersion|Resources|AWS::|Description)")
3637
val AWSTemplateCaseInsensitiveKeyWordsRegex = Regex("(cloudformation|cfn|template|description)")
@@ -77,6 +78,7 @@ object CodeWhispererConstants {
7778
const val TOTAL_MILLIS_IN_SECOND = 1000
7879
const val TOTAL_SECONDS_IN_MINUTE: Long = 60L
7980
const val ACCOUNTLESS_START_URL = "accountless"
81+
const val FEATURE_CONFIG_POLL_INTERVAL_IN_MS: Long = 30 * 60 * 1000L // 30 mins
8082

8183
// Date when Accountless is not supported
8284
val EXPIRE_DATE = SimpleDateFormat("yyyy-MM-dd").parse("2023-01-31")

jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import com.intellij.openapi.vfs.VfsUtil
1313
import com.intellij.openapi.vfs.VirtualFile
1414
import kotlinx.coroutines.yield
1515
import software.amazon.awssdk.services.codewhispererruntime.model.Completion
16+
import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
1617
import software.aws.toolkits.jetbrains.core.credentials.AwsBearerTokenConnection
1718
import software.aws.toolkits.jetbrains.core.credentials.ManagedBearerSsoConnection
1819
import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnection
@@ -35,6 +36,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.learn.LearnCodeWhi
3536
import software.aws.toolkits.jetbrains.services.codewhisperer.model.Chunk
3637
import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererService
3738
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.isTelemetryEnabled
39+
import software.aws.toolkits.jetbrains.settings.AwsSettings
3840
import software.aws.toolkits.jetbrains.utils.notifyError
3941
import software.aws.toolkits.jetbrains.utils.notifyInfo
4042
import software.aws.toolkits.jetbrains.utils.notifyWarn
@@ -274,6 +276,13 @@ object CodeWhispererUtil {
274276
val filename = (editor as EditorImpl).virtualFile?.name ?: return null
275277
return taskTypeToFilename.filter { filename.startsWith(it.value) }.keys.firstOrNull()
276278
}
279+
280+
fun getTelemetryOptOutPreference() =
281+
if (AwsSettings.getInstance().isTelemetryEnabled) {
282+
OptOutPreference.OPTIN
283+
} else {
284+
OptOutPreference.OPTOUT
285+
}
277286
}
278287

279288
enum class CaretMovement {

0 commit comments

Comments
 (0)