diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt index 1dfd5d556be..6ba3e038d5d 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt @@ -56,7 +56,6 @@ import software.aws.toolkits.jetbrains.services.telemetry.ClientMetadata import software.aws.toolkits.telemetry.CodewhispererCompletionType import software.aws.toolkits.telemetry.CodewhispererSuggestionState import java.time.Instant -import java.util.concurrent.TimeUnit import kotlin.reflect.KProperty0 import kotlin.reflect.jvm.isAccessible @@ -114,7 +113,8 @@ interface CodeWhispererClientAdaptor : Disposable { requestId: String, language: CodeWhispererProgrammingLanguage, customizationArn: String, - modificationPercentage: Double, + acceptedCharacterCount: Int, + unmodifiedAcceptedTokenCount: Int, ): SendTelemetryEventResponse fun sendCodeScanTelemetry( @@ -305,13 +305,14 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW val programmingLanguage = fileContext.programmingLanguage var e2eLatency = requestContext.latencyContext.getCodeWhispererEndToEndLatency() - // When we send a userTriggerDecision of Empty or Discard, we set the time users see the first - // suggestion to be now. - if (e2eLatency < 0) { - e2eLatency = TimeUnit.NANOSECONDS.toMillis( - System.nanoTime() - requestContext.latencyContext.codewhispererEndToEndStart - ).toDouble() + // When we send a userTriggerDecision for neither Accept nor Reject, service side should not use this value + // and client side will set this value to 0.0. + if (suggestionState != CodewhispererSuggestionState.Accept && + suggestionState != CodewhispererSuggestionState.Reject + ) { + e2eLatency = 0.0 } + return bearerClient().sendTelemetryEvent { requestBuilder -> requestBuilder.telemetryEvent { telemetryEventBuilder -> telemetryEventBuilder.userTriggerDecisionEvent { @@ -321,6 +322,9 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW it.sessionId(responseContext.sessionId) it.recommendationLatencyMilliseconds(e2eLatency) it.triggerToResponseLatencyMilliseconds(requestContext.latencyContext.paginationFirstCompletionTime) + it.perceivedLatencyMilliseconds( + requestContext.latencyContext.getPerceivedLatency(requestContext.triggerTypeInfo.triggerType) + ) it.suggestionState(suggestionState.toCodeWhispererSdkType()) it.timestamp(Instant.now()) it.suggestionReferenceCount(suggestionReferenceCount) @@ -360,7 +364,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW requestId: String, language: CodeWhispererProgrammingLanguage, customizationArn: String, - modificationPercentage: Double, + acceptedCharacterCount: Int, + unmodifiedAcceptedTokenCount: Int, ): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder -> requestBuilder.telemetryEvent { telemetryEventBuilder -> telemetryEventBuilder.userModificationEvent { @@ -370,8 +375,11 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW languageBuilder.languageName(language.toCodeWhispererRuntimeLanguage().languageId) } it.customizationArn(customizationArn) - it.modificationPercentage(modificationPercentage) + // deprecated field, service side should not use this % anymore + it.modificationPercentage(0.0) it.timestamp(Instant.now()) + it.acceptedCharacterCount(acceptedCharacterCount) + it.unmodifiedAcceptedCharacterCount(unmodifiedAcceptedTokenCount) } } requestBuilder.optOutPreference(getTelemetryOptOutPreference()) diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt index cf465134162..828f09b3833 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.GenerateComple import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnection import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.PayloadContext import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage +import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererAutoTriggerService import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererAutomatedTriggerType import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestContext import software.aws.toolkits.jetbrains.services.codewhisperer.service.ResponseContext @@ -198,6 +199,18 @@ data class LatencyContext( fun getCodeWhispererPreprocessingLatency() = TimeUnit.NANOSECONDS.toMillis( codewhispererPreprocessingEnd - codewhispererPreprocessingStart ).toDouble() + + // For auto-trigger it's from the time when last char typed + // for manual-trigger it's from the time when last trigger action happened(alt + c) + fun getPerceivedLatency(triggerType: CodewhispererTriggerType) = + if (triggerType == CodewhispererTriggerType.OnDemand) { + getCodeWhispererEndToEndLatency() + } else { + ( + TimeUnit.NANOSECONDS.toMillis(codewhispererEndToEndEnd) - + CodeWhispererAutoTriggerService.getInstance().timeAtLastCharTyped.toEpochMilli() + ).toDouble() + } } data class TryExampleRowContext( diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererAutoTriggerService.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererAutoTriggerService.kt index 47c35c851d6..b1579ec1df8 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererAutoTriggerService.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererAutoTriggerService.kt @@ -42,6 +42,8 @@ class CodeWhispererAutoTriggerService : CodeWhispererAutoTriggerHandler, Disposa private var lastInvocationTime: Instant? = null private var lastInvocationLineNum: Int? = null + var timeAtLastCharTyped: Instant = Instant.now() + private set init { scheduleReset() @@ -54,6 +56,7 @@ class CodeWhispererAutoTriggerService : CodeWhispererAutoTriggerHandler, Disposa // a util wrapper fun tryInvokeAutoTrigger(editor: Editor, triggerType: CodeWhispererAutomatedTriggerType): Job? { // only needed for Classifier group, thus calculate it lazily + timeAtLastCharTyped = Instant.now() val classifierResult: ClassifierResult by lazy { shouldTriggerClassifier(editor, triggerType.telemetryType) } val language = runReadAction { FileDocumentManager.getInstance().getFile(editor.document)?.programmingLanguage() diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt index 7e9a48b4baa..c0c325c88b0 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt @@ -32,6 +32,14 @@ class CodeWhispererFeatureConfigService { featureConfigs[it.feature()] = FeatureContext(it.feature(), it.variation(), it.value()) } + // Only apply new auto-trigger UX to BID users + val isNewAutoTriggerUX = getNewAutoTriggerUX() + if (isNewAutoTriggerUX) { + calculateIfIamIdentityCenterConnection(project) { + featureConfigs.remove(NEW_AUTO_TRIGGER_UX) + } + } + val customizationArnOverride = featureConfigs[CUSTOMIZATION_ARN_OVERRIDE_NAME]?.value?.stringValue() if (customizationArnOverride != null) { // Double check if server-side wrongly returns a customizationArn to BID users @@ -84,20 +92,24 @@ class CodeWhispererFeatureConfigService { fun getCustomizationArnOverride(): String = getFeatureValueForKey(CUSTOMIZATION_ARN_OVERRIDE_NAME).stringValue() + fun getNewAutoTriggerUX(): Boolean = getFeatureValueForKey(NEW_AUTO_TRIGGER_UX).boolValue() + // Get the feature value for the given key. - // In case of a misconfiguration, it will return a default feature value of Boolean true. + // In case of a misconfiguration, it will return a default feature value of Boolean false. private fun getFeatureValueForKey(name: String): FeatureValue = featureConfigs[name]?.value ?: FEATURE_DEFINITIONS[name]?.value - ?: FeatureValue.builder().boolValue(true).build() + ?: FeatureValue.builder().boolValue(false).build() companion object { fun getInstance(): CodeWhispererFeatureConfigService = service() private const val TEST_FEATURE_NAME = "testFeature" private const val DATA_COLLECTION_FEATURE = "IDEProjectContextDataCollection" const val CUSTOMIZATION_ARN_OVERRIDE_NAME = "customizationArnOverride" + private const val NEW_AUTO_TRIGGER_UX = "newAutoTriggerUX" private val LOG = getLogger() // TODO: add real feature later + // Also serve as default values in case server-side config isn't there yet internal val FEATURE_DEFINITIONS = mapOf( TEST_FEATURE_NAME to FeatureContext( TEST_FEATURE_NAME, @@ -109,7 +121,12 @@ class CodeWhispererFeatureConfigService { CUSTOMIZATION_ARN_OVERRIDE_NAME, "customizationARN", FeatureValue.builder().stringValue("").build() - ) + ), + NEW_AUTO_TRIGGER_UX to FeatureContext( + NEW_AUTO_TRIGGER_UX, + "CONTROL", + FeatureValue.builder().boolValue(false).build() + ), ) } } diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt index bdfb45cd57c..026e2101026 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt @@ -244,7 +244,8 @@ class CodeWhispererService(private val cs: CoroutineScope) : Disposable { val sessionId = response.sdkHttpResponse().headers().getOrDefault(KET_SESSION_ID, listOf(requestId))[0] if (requestCount == 1) { requestContext.latencyContext.codewhispererPostprocessingStart = System.nanoTime() - requestContext.latencyContext.paginationFirstCompletionTime = latency + requestContext.latencyContext.paginationFirstCompletionTime = + (endTime - requestContext.latencyContext.codewhispererEndToEndStart).toDouble() requestContext.latencyContext.firstRequestId = requestId CodeWhispererInvocationStatus.getInstance().setInvocationSessionId(sessionId) } diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt index aefa1b51dad..3d55707e216 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt @@ -31,6 +31,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispe import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererService import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.TOTAL_SECONDS_IN_MINUTE import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCodeWhispererStartUrl +import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getUnmodifiedAcceptedCharsCount import software.aws.toolkits.jetbrains.services.codewhisperer.util.runIfIdcConnectionOrTelemetryEnabled import software.aws.toolkits.telemetry.CodewhispererTelemetry import java.time.Duration @@ -40,6 +41,7 @@ import java.util.concurrent.atomic.AtomicInteger import kotlin.math.roundToLong // TODO: reset code coverage calculator on logging out connection? +// TODO: rename "Tokens" to "Characters", and many more renames in this file abstract class CodeWhispererCodeCoverageTracker( private val project: Project, private val timeWindowInSec: Long, @@ -49,22 +51,22 @@ abstract class CodeWhispererCodeCoverageTracker( private val myServiceInvocationCount: AtomicInteger, ) : Disposable { val percentage: Long? - get() = if (totalTokensSize != 0L) calculatePercentage(rawAcceptedTokenSize, totalTokensSize) else null - val acceptedTokensSize: Long + get() = if (totalCharsCount != 0L) calculatePercentage(acceptedCharsCount, totalCharsCount) else null + val unmodifiedAcceptedCharsCount: Long get() = fileToTokens.map { - it.value.acceptedTokens.get() + it.value.unmodifiedAcceptedChars.get() }.fold(0) { acc, next -> acc + next } - val totalTokensSize: Long + val totalCharsCount: Long get() = fileToTokens.map { - it.value.totalTokens.get() + it.value.totalChars.get() }.fold(0) { acc, next -> acc + next } - private val rawAcceptedTokenSize: Long + private val acceptedCharsCount: Long get() = fileToTokens.map { - it.value.rawAcceptedTokens.get() + it.value.acceptedChars.get() }.fold(0) { acc, next -> acc + next } @@ -93,10 +95,10 @@ abstract class CodeWhispererCodeCoverageTracker( rangeMarker.putUserData(KEY_REMAINING_RECOMMENDATION, originalRecommendation) runReadAction { // also increment total tokens because accepted tokens are part of it - incrementTotalTokens(rangeMarker.document, originalRecommendation.length) + incrementTotalCharsCount(rangeMarker.document, originalRecommendation.length) // avoid counting CodeWhisperer inserted suggestion twice in total tokens if (rangeMarker.textRange.length in 2..49 && originalRecommendation.trim().isNotEmpty()) { - incrementTotalTokens(rangeMarker.document, -rangeMarker.textRange.length) + incrementTotalCharsCount(rangeMarker.document, -rangeMarker.textRange.length) } } } @@ -133,12 +135,12 @@ abstract class CodeWhispererCodeCoverageTracker( // when event is auto closing [{(', there will be 2 separated events, both count as 1 char increase in total chars val text = event.newFragment.toString() if ((event.newLength == 1 && event.oldLength == 0) || (text.startsWith('\n') && text.trim().isEmpty())) { - incrementTotalTokens(event.document, 1) + incrementTotalCharsCount(event.document, 1) return } else if (event.newLength < 50 && text.trim().isNotEmpty()) { // count doc changes from <50 multi character input as total user written code // ignore all white space changes, this usually comes from IntelliJ formatting - incrementTotalTokens(event.document, event.newLength) + incrementTotalCharsCount(event.document, event.newLength) } } @@ -146,19 +148,6 @@ abstract class CodeWhispererCodeCoverageTracker( rangeMarker.range?.let { myRange -> rangeMarker.document.getText(myRange) } } - // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace), - // and thus the unmodified part of recommendation length can be deducted/approximated - // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3 - // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8 - // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1 - internal fun getAcceptedTokensDelta(originalRecommendation: String, modifiedRecommendation: String): Int { - val editDistance = getEditDistance(modifiedRecommendation, originalRecommendation).toInt() - return maxOf(originalRecommendation.length, modifiedRecommendation.length) - editDistance - } - - protected open fun getEditDistance(modifiedString: String, originalString: String): Double = - levenshteinChecker.distance(modifiedString, originalString) - private fun flush() { try { if (isTelemetryEnabled()) emitCodeWhispererCodeContribution() @@ -174,21 +163,21 @@ abstract class CodeWhispererCodeCoverageTracker( } } - private fun incrementAcceptedTokens(document: Document, delta: Int) { + private fun incrementUnmodifiedAcceptedCharsCount(document: Document, delta: Int) { val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() } - tokens.acceptedTokens.addAndGet(delta) + tokens.unmodifiedAcceptedChars.addAndGet(delta) } - private fun incrementRawAcceptedTokens(document: Document, delta: Int) { + private fun incrementAcceptedCharsCount(document: Document, delta: Int) { val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() } - tokens.rawAcceptedTokens.addAndGet(delta) + tokens.acceptedChars.addAndGet(delta) } - private fun incrementTotalTokens(document: Document, delta: Int) { + private fun incrementTotalCharsCount(document: Document, delta: Int) { val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() } tokens.apply { - totalTokens.addAndGet(delta) - if (totalTokens.get() < 0) totalTokens.set(0) + totalChars.addAndGet(delta) + if (totalChars.get() < 0) totalChars.set(0) } } @@ -217,10 +206,10 @@ abstract class CodeWhispererCodeCoverageTracker( } return@forEach } - val delta = getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) + val unmodifiedRecommendationLength = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) runReadAction { - incrementRawAcceptedTokens(rangeMarker.document, originalRecommendation.length) - incrementAcceptedTokens(rangeMarker.document, delta) + incrementAcceptedCharsCount(rangeMarker.document, originalRecommendation.length) + incrementUnmodifiedAcceptedCharsCount(rangeMarker.document, unmodifiedRecommendationLength) } } val customizationArn: String? = CodeWhispererModelConfigurator.getInstance().activeCustomization(project)?.arn @@ -231,9 +220,9 @@ abstract class CodeWhispererCodeCoverageTracker( val response = CodeWhispererClientAdaptor.getInstance(project).sendCodePercentageTelemetry( language, customizationArn, - rawAcceptedTokenSize, - totalTokensSize, - acceptedTokensSize + acceptedCharsCount, + totalCharsCount, + unmodifiedAcceptedCharsCount ) LOG.debug { "Successfully sent code percentage telemetry. RequestId: ${response.responseMetadata().requestId()}" } } catch (e: Exception) { @@ -248,11 +237,11 @@ abstract class CodeWhispererCodeCoverageTracker( percentage?.let { percentage -> CodewhispererTelemetry.codePercentage( project = null, - codewhispererAcceptedTokens = acceptedTokensSize, - codewhispererSuggestedTokens = rawAcceptedTokenSize, + codewhispererAcceptedTokens = unmodifiedAcceptedCharsCount, + codewhispererSuggestedTokens = acceptedCharsCount, codewhispererLanguage = language.toTelemetryType(), codewhispererPercentage = percentage, - codewhispererTotalTokens = totalTokensSize, + codewhispererTotalTokens = totalCharsCount, successCount = myServiceInvocationCount.get().toLong(), codewhispererCustomizationArn = customizationArn, credentialStartUrl = getCodeWhispererStartUrl(project) @@ -277,7 +266,7 @@ abstract class CodeWhispererCodeCoverageTracker( companion object { @JvmStatic - protected val levenshteinChecker = Levenshtein() + val levenshteinChecker = Levenshtein() private const val REMAINING_RECOMMENDATION = "remainingRecommendation" private val KEY_REMAINING_RECOMMENDATION = Key(REMAINING_RECOMMENDATION) private val LOG = getLogger() @@ -311,14 +300,8 @@ class DefaultCodeWhispererCodeCoverageTracker(project: Project, language: CodeWh AtomicInteger(0) ) -class CodeCoverageTokens(totalTokens: Int = 0, acceptedTokens: Int = 0, rawAcceptedTokens: Int = 0) { - val totalTokens: AtomicInteger - val acceptedTokens: AtomicInteger - val rawAcceptedTokens: AtomicInteger - - init { - this.totalTokens = AtomicInteger(totalTokens) - this.acceptedTokens = AtomicInteger(acceptedTokens) - this.rawAcceptedTokens = AtomicInteger(rawAcceptedTokens) - } +class CodeCoverageTokens(totalChars: Int = 0, unmodifiedAcceptedChars: Int = 0, acceptedChars: Int = 0) { + val totalChars = AtomicInteger(totalChars) + val unmodifiedAcceptedChars = AtomicInteger(unmodifiedAcceptedChars) + val acceptedChars = AtomicInteger(acceptedChars) } diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererUserModificationTracker.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererUserModificationTracker.kt index 84e93fb8198..ec45aa9ddf2 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererUserModificationTracker.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererUserModificationTracker.kt @@ -16,16 +16,20 @@ import com.intellij.util.Alarm import com.intellij.util.AlarmFactory import info.debatty.java.stringsimilarity.Levenshtein import org.assertj.core.util.VisibleForTesting +import software.amazon.awssdk.services.codewhispererruntime.model.CodeWhispererRuntimeException import software.aws.toolkits.core.utils.debug import software.aws.toolkits.core.utils.getLogger import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnection import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator +import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererLanguageManager import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage import software.aws.toolkits.jetbrains.services.codewhisperer.language.languages.CodeWhispererUnknownLanguage import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhispererSettings import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getConnectionStartUrl +import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getUnmodifiedAcceptedCharsCount +import software.aws.toolkits.jetbrains.services.codewhisperer.util.runIfIdcConnectionOrTelemetryEnabled import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.InsertedCodeModificationEntry import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl import software.aws.toolkits.jetbrains.settings.AwsSettings @@ -152,28 +156,18 @@ class CodeWhispererUserModificationTracker(private val project: Project) : Dispo if (file == null || (!file.isValid)) { sendModificationTelemetry(acceptedSuggestion, null) - // temp remove event sent as further discussion needed for metric calculation - // sendUserModificationTelemetryToServiceAPI(acceptedSuggestion, 1.0) + sendUserModificationTelemetryToServiceAPI(acceptedSuggestion) } else { - try { - /** - * this try-catch is to check if the offsets are valid since the method does not return null - */ - val document = runReadAction { - FileDocumentManager.getInstance().getDocument(file) - } - val currentString = document?.getText( - TextRange(acceptedSuggestion.range.startOffset, acceptedSuggestion.range.endOffset) - ) - val modificationPercentage = checkDiff(currentString?.trim(), acceptedSuggestion.suggestion.trim()) - sendModificationTelemetry(acceptedSuggestion, modificationPercentage) - // temp remove event sent as further discussion needed for metric calculation - // sendUserModificationTelemetryToServiceAPI(acceptedSuggestion, modificationPercentage) - } catch (e: Exception) { - sendModificationTelemetry(acceptedSuggestion, null) - // temp remove event sent as further discussion needed for metric calculation - // sendUserModificationTelemetryToServiceAPI(acceptedSuggestion, 1.0) + // Will remove this later when we truly don't need toolkit user modification telemetry anymore + val document = runReadAction { + FileDocumentManager.getInstance().getDocument(file) } + val currentString = document?.getText( + TextRange(acceptedSuggestion.range.startOffset, acceptedSuggestion.range.endOffset) + ) + val modificationPercentage = checkDiff(currentString?.trim(), acceptedSuggestion.suggestion.trim()) + sendModificationTelemetry(acceptedSuggestion, modificationPercentage) + sendUserModificationTelemetryToServiceAPI(acceptedSuggestion) } } @@ -237,47 +231,37 @@ class CodeWhispererUserModificationTracker(private val project: Project) : Dispo } } -// temp disable user modfication event for further discussion on metric calculation -// private fun sendUserModificationTelemetryToServiceAPI( -// suggestion: AcceptedSuggestionEntry, -// modificationPercentage: Double -// ) { -// calculateIfIamIdentityCenterConnection(project) { -// val response = try { -// CodeWhispererClientAdaptor.getInstance(project) -// .sendUserModificationTelemetry( -// suggestion.sessionId, -// suggestion.requestId, -// suggestion.vFile?.let { CodeWhispererLanguageManager.getInstance().getLanguage(suggestion.vFile) }, -// CodeWhispererModelConfigurator.getInstance().activeCustomization(project)?.arn.orEmpty(), -// modificationPercentage -// ) -// } catch (e: Exception) { -// when (e) { -// is CodeWhispererRuntimeException -> { -// LOG.info(e) { -// "Failed to send code scan telemetry with Code Whisperer Runtime Exception" -// } -// } -// -// is CodeWhispererException -> { -// LOG.info(e) { -// "Failed to send code scan telemetry with Code Whisperer Exception" -// } -// } -// -// else -> { -// LOG.info(e) { "Failed to send user modification telemetry." } -// } -// } -// null -// } -// -// response?.let { -// LOG.debug { "Successfully sent user modification telemetry. RequestId: ${it.responseMetadata().requestId()}" } -// } -// } -// } + private fun sendUserModificationTelemetryToServiceAPI( + suggestion: AcceptedSuggestionEntry, + ) { + runIfIdcConnectionOrTelemetryEnabled(project) { + try { + // should be impossible from the caller logic + if (suggestion.vFile == null) return@runIfIdcConnectionOrTelemetryEnabled + val document = runReadAction { + FileDocumentManager.getInstance().getDocument(suggestion.vFile) + } + val modifiedSuggestion = document?.getText( + TextRange(suggestion.range.startOffset, suggestion.range.endOffset) + ) ?: "" + val response = CodeWhispererClientAdaptor.getInstance(project) + .sendUserModificationTelemetry( + suggestion.sessionId, + suggestion.requestId, + CodeWhispererLanguageManager.getInstance().getLanguage(suggestion.vFile), + CodeWhispererModelConfigurator.getInstance().activeCustomization(project)?.arn.orEmpty(), + suggestion.suggestion.length, + getUnmodifiedAcceptedCharsCount(suggestion.suggestion, modifiedSuggestion) + ) + LOG.debug { "Successfully sent user modification telemetry. RequestId: ${response.responseMetadata().requestId()}" } + } catch (e: Exception) { + val requestId = if (e is CodeWhispererRuntimeException) e.requestId() else null + LOG.debug { + "Failed to send user modification telemetry. RequestId: $requestId, ErrorMessage: ${e.message}" + } + } + } + } companion object { private val DEFAULT_CHECK_INTERVAL = Duration.ofMinutes(1) diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt index 220ef0200d4..d7f5ce08f3e 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt @@ -34,6 +34,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhisp import software.aws.toolkits.jetbrains.services.codewhisperer.learn.LearnCodeWhispererManager.Companion.taskTypeToFilename import software.aws.toolkits.jetbrains.services.codewhisperer.model.Chunk import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererService +import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererCodeCoverageTracker.Companion.levenshteinChecker import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.isTelemetryEnabled import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.CrossFile.NUMBER_OF_CHUNK_TO_FETCH import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.CrossFile.NUMBER_OF_LINE_IN_CHUNK @@ -307,6 +308,19 @@ object CodeWhispererUtil { } } } + + // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace), + // and thus the unmodified part of recommendation length can be deducted/approximated + // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3 + // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8 + // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1 + fun getUnmodifiedAcceptedCharsCount(originalRecommendation: String, modifiedRecommendation: String): Int { + val editDistance = getEditDistance(modifiedRecommendation, originalRecommendation).toInt() + return maxOf(originalRecommendation.length, modifiedRecommendation.length) - editDistance + } + + private fun getEditDistance(modifiedString: String, originalString: String): Double = + levenshteinChecker.distance(modifiedString, originalString) } enum class CaretMovement { diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererClientAdaptorTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererClientAdaptorTest.kt index e76a34d2e45..c98639cd067 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererClientAdaptorTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererClientAdaptorTest.kt @@ -402,7 +402,7 @@ class CodeWhispererClientAdaptorTest { @Test fun `sendTelemetryEvent for userModification respects telemetry optin status`() { sendTelemetryEventOptOutCheckHelper { - sut.sendUserModificationTelemetry(aString(), aString(), aProgrammingLanguage(), aString(), 0.0) + sut.sendUserModificationTelemetry(aString(), aString(), aProgrammingLanguage(), aString(), 0, 0) } } diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt index 61ac7b68426..a82f8700eb4 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt @@ -253,7 +253,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov val captor = argumentCaptor() verify(sut, Times(1)).documentChanged(captor.capture()) assertThat(captor.firstValue.newFragment.toString()).isEqualTo(keystrokeInput) - assertThat(sut.totalTokensSize).isEqualTo(keystrokeInput.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(keystrokeInput.length.toLong()) } @Test @@ -271,7 +271,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov val captor = argumentCaptor() verify(sut, Times(1)).documentChanged(captor.capture()) assertThat(captor.firstValue.newFragment.toString()).isEqualTo(pythonTestLeftContext) - assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong()) val anotherCode = "(x, y):".repeat(8) runInEdtAndWait { @@ -279,7 +279,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov fixture.editor.appendString(anotherCode) } } - assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong()) } @Test @@ -293,7 +293,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut sut.activateTrackerIfNotActive() - assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong()) runInEdtAndWait { fixture.editor.caretModel.primaryCaret.moveToOffset(fixture.editor.document.textLength) @@ -302,7 +302,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov } } - assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong()) } @Test @@ -315,7 +315,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov ) CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut sut.activateTrackerIfNotActive() - assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong()) runInEdtAndWait { WriteCommandAction.runWriteCommandAction(project) { @@ -323,7 +323,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov } } - assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length + 1L) + assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length + 1L) } @Test @@ -366,36 +366,14 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov sut.activateTrackerIfNotActive() assertThat(sut.activeRequestCount()).isEqualTo(1) - assertThat(sut.acceptedTokensSize).isEqualTo("bar".length.toLong()) - assertThat(sut.totalTokensSize).isEqualTo("foobar".length.toLong()) + assertThat(sut.unmodifiedAcceptedCharsCount).isEqualTo("bar".length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo("foobar".length.toLong()) sut.forceTrackerFlush() assertThat(sut.activeRequestCount()).isEqualTo(1) - assertThat(sut.acceptedTokensSize).isEqualTo(0) - assertThat(sut.totalTokensSize).isEqualTo(0) - } - - @Test - fun `test when rangeMarker is not vaild, acceptedToken will not be updated`() { - // when user delete whole recommendation, rangeMarker will be isValid = false - val rangeMarkerMock: RangeMarker = mock() - whenever(rangeMarkerMock.isValid).thenReturn(false) - sut = spy( - TestCodePercentageTracker( - project, - TOTAL_SECONDS_IN_MINUTE, - CodeWhispererPython.INSTANCE, - mutableListOf(rangeMarkerMock), - ) - ) { - onGeneric { getAcceptedTokensDelta(any(), any()) } doReturn 100 - } - - sut.activateTrackerIfNotActive() - sut.forceTrackerFlush() - - verify(sut, Times(0)).getAcceptedTokensDelta(any(), any()) + assertThat(sut.unmodifiedAcceptedCharsCount).isEqualTo(0) + assertThat(sut.totalCharsCount).isEqualTo(0) } @Test @@ -430,12 +408,11 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov TOTAL_SECONDS_IN_MINUTE, CodeWhispererPython.INSTANCE, mutableListOf(rangeMarkerMock1), - mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalTokens = 100, acceptedTokens = 0, rawAcceptedTokens = 0)), + mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalChars = 100, unmodifiedAcceptedChars = 0, acceptedChars = 0)), 1 ) ) { onGeneric { extractRangeMarkerString(any()) } doReturn "fou" - onGeneric { getAcceptedTokensDelta(any(), any()) } doReturn 1 } sut.emitCodeWhispererCodeContribution() @@ -445,8 +422,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov metricCaptor.allValues, CODE_PERCENTAGE, 1, - CWSPR_PERCENTAGE to "3", - CWSPR_ACCEPTED_TOKENS to "1", + CWSPR_ACCEPTED_TOKENS to "2", CWSPR_RAW_ACCEPTED_TOKENS to "3", CWSPR_TOTAL_TOKENS to "100", ) @@ -464,51 +440,6 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov verify(sut, Times(0)).emitCodeWhispererCodeContribution() } - @Test - fun `test getAcceptedTokensDelta()`() { - val tracker = TestCodePercentageTracker(project, TOTAL_SECONDS_IN_MINUTE, CodeWhispererPython.INSTANCE) - var originalRecommendation = "foo" - var modifiedRecommendation = "fou" - var delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo(2) - - originalRecommendation = "foo" - modifiedRecommendation = "f11111oo" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo(3) - - originalRecommendation = "foo" - modifiedRecommendation = "fo" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo(2) - - originalRecommendation = "helloworld" - modifiedRecommendation = "HelloWorld" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo("helloworld".length - 2) - - originalRecommendation = "helloworld" - modifiedRecommendation = "World" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo("helloworld".length - "hello".length - 1) - - originalRecommendation = "CodeWhisperer" - modifiedRecommendation = "CODE" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo(1) - - originalRecommendation = "CodeWhisperer" - modifiedRecommendation = "codewhispererISBEST" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo("CodeWhisperer".length - 2) - - val pythonCommentAddedByUser = "\"\"\"we don't count this comment as generated by CodeWhisperer\"\"\"\n" - originalRecommendation = "x, y):\n\treturn x + y" - modifiedRecommendation = "x, y):\n$pythonCommentAddedByUser\treturn x + y" - delta = tracker.getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation) - assertThat(delta).isEqualTo(originalRecommendation.length) - } - @Test fun `test flush() won't emit telemetry when users are not editing the document (totalTokens == 0)`() { sut = TestCodePercentageTracker(project, TOTAL_SECONDS_IN_MINUTE, CodeWhispererPython.INSTANCE) @@ -557,7 +488,7 @@ internal class CodeWhispererCodeCoverageTrackerTestJava : CodeWhispererCodeCover project, TOTAL_SECONDS_IN_MINUTE, language = CodeWhispererJava.INSTANCE, - codeCoverageTokens = mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalTokens = codeNeedToBeReformatted.length)) + codeCoverageTokens = mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalChars = codeNeedToBeReformatted.length)) ) ) CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererJava.INSTANCE] = sut @@ -568,7 +499,7 @@ internal class CodeWhispererCodeCoverageTrackerTestJava : CodeWhispererCodeCover } // reformat should fire documentChanged events, but tracker should not update token from these events verify(sut, atLeastOnce()).documentChanged(any()) - assertThat(sut.totalTokensSize).isEqualTo(codeNeedToBeReformatted.length.toLong()) + assertThat(sut.totalCharsCount).isEqualTo(codeNeedToBeReformatted.length.toLong()) val formatted = """ class Answer { diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUtilTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUtilTest.kt index 00b3a8cd975..300eaaa1e07 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUtilTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUtilTest.kt @@ -23,6 +23,7 @@ import software.aws.toolkits.jetbrains.core.credentials.sono.SONO_URL import software.aws.toolkits.jetbrains.core.region.MockRegionProviderRule import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCompletionType import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference +import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getUnmodifiedAcceptedCharsCount import software.aws.toolkits.jetbrains.services.codewhisperer.util.runIfIdcConnectionOrTelemetryEnabled import software.aws.toolkits.jetbrains.services.codewhisperer.util.toCodeChunk import software.aws.toolkits.jetbrains.settings.AwsSettings @@ -192,4 +193,48 @@ class CodeWhispererUtilTest { assertThat(AwsSettings.getInstance().isTelemetryEnabled).isFalse assertThat(getTelemetryOptOutPreference()).isEqualTo(OptOutPreference.OPTOUT) } + + @Test + fun `test getUnmodifiedAcceptedCharsCount()`() { + var originalRecommendation = "foo" + var modifiedRecommendation = "fou" + var unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo(2) + + originalRecommendation = "foo" + modifiedRecommendation = "f11111oo" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo(3) + + originalRecommendation = "foo" + modifiedRecommendation = "fo" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo(2) + + originalRecommendation = "helloworld" + modifiedRecommendation = "HelloWorld" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo("helloworld".length - 2) + + originalRecommendation = "helloworld" + modifiedRecommendation = "World" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo("helloworld".length - "hello".length - 1) + + originalRecommendation = "CodeWhisperer" + modifiedRecommendation = "CODE" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo(1) + + originalRecommendation = "CodeWhisperer" + modifiedRecommendation = "codewhispererISBEST" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo("CodeWhisperer".length - 2) + + val pythonCommentAddedByUser = "\"\"\"we don't count this comment as generated by CodeWhisperer\"\"\"\n" + originalRecommendation = "x, y):\n\treturn x + y" + modifiedRecommendation = "x, y):\n$pythonCommentAddedByUser\treturn x + y" + unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation) + assertThat(unmodifiedCharsCount).isEqualTo(originalRecommendation.length) + } } diff --git a/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json b/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json index f8f4a99f5d1..0f29c541040 100644 --- a/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json +++ b/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json @@ -1963,7 +1963,9 @@ "requestId", "programmingLanguage", "modificationPercentage", - "timestamp" + "timestamp", + "acceptedCharacterCount", + "unmodifiedAcceptedCharacterCount" ], "members":{ "sessionId":{"shape":"UUID"}, @@ -1971,7 +1973,9 @@ "programmingLanguage":{"shape":"ProgrammingLanguage"}, "modificationPercentage":{"shape":"Double"}, "customizationArn":{"shape":"CustomizationArn"}, - "timestamp":{"shape":"Timestamp"} + "timestamp":{"shape":"Timestamp"}, + "acceptedCharacterCount":{"shape":"PrimitiveInteger"}, + "unmodifiedAcceptedCharacterCount":{"shape":"PrimitiveInteger"} } }, "UserTriggerDecisionEvent":{ @@ -1997,7 +2001,9 @@ "triggerToResponseLatencyMilliseconds":{"shape":"Double"}, "suggestionReferenceCount":{"shape":"PrimitiveInteger"}, "generatedLine":{"shape":"PrimitiveInteger"}, - "numberOfRecommendations":{"shape":"PrimitiveInteger"} + "numberOfRecommendations":{"shape":"PrimitiveInteger"}, + "perceivedLatencyMilliseconds":{"shape":"Double"}, + "acceptedCharacterCount":{"shape":"PrimitiveInteger"} } }, "ValidationException":{