Skip to content

Commit f60dc5e

Browse files
Will-ShaoHuaWill-ShaoHua
authored
add new fields in UserModification metrics (#4770)
1 parent 5dd1a5a commit f60dc5e

File tree

4 files changed

+108
-25
lines changed

4 files changed

+108
-25
lines changed

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ mockitoKotlin = "5.4.0"
2525
mockk = "1.13.10"
2626
nimbus-jose-jwt = "9.40"
2727
node-gradle = "7.0.2"
28-
telemetryGenerator = "1.0.232"
28+
telemetryGenerator = "1.0.236"
2929
testLogger = "4.0.0"
3030
testRetry = "1.5.10"
3131
# test-only; platform provides slf4j transitively at runtime. <233, 1.7.36; >=233, 2.0.9

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeTransformTelemetryManager.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,9 @@ class CodeTransformTelemetryManager(private val project: Project) {
8484
codeTransformRunTimeLatency = calculateTotalLatency(startTime, Instant.now()),
8585
)
8686

87+
@Suppress("UNUSED_PARAMETER")
8788
fun apiError(errorMessage: String, apiName: CodeTransformApiNames, jobId: String?) = CodetransformTelemetry.logApiError(
88-
codeTransformApiErrorMessage = errorMessage,
89-
codeTransformApiNames = apiName,
89+
reason = errorMessage,
9090
codeTransformSessionId = sessionId,
9191
codeTransformJobId = jobId,
9292
)
@@ -302,7 +302,7 @@ class CodeTransformTelemetryManager(private val project: Project) {
302302
)
303303

304304
fun error(errorMessage: String) = CodetransformTelemetry.logGeneralError(
305-
codeTransformApiErrorMessage = errorMessage,
305+
reason = errorMessage,
306306
codeTransformSessionId = sessionId,
307307
)
308308

‎plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererUserModificationTracker.kt‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,21 @@ data class AcceptedSuggestionEntry(
5757
val connection: ToolkitConnection?
5858
) : UserModificationTrackingEntry
5959

60+
data class CodeInsertionDiff(
61+
val original: String,
62+
val modified: String,
63+
val diff: Double
64+
)
65+
66+
fun CodeInsertionDiff?.percentage(): Double = when {
67+
this == null -> 1.0
68+
69+
// TODO: should revisit this case
70+
original.isEmpty() || modified.isEmpty() -> 1.0
71+
72+
else -> min(1.0, (diff / original.length))
73+
}
74+
6075
@Service(Service.Level.PROJECT)
6176
class CodeWhispererUserModificationTracker(private val project: Project) : Disposable {
6277
private val acceptedSuggestions = LinkedBlockingDeque<UserModificationTrackingEntry>(DEFAULT_MAX_QUEUE_SIZE)
@@ -129,15 +144,15 @@ class CodeWhispererUserModificationTracker(private val project: Project) : Dispo
129144
val modificationPercentage = checkDiff(currentString?.trim(), insertedCode.originalString.trim())
130145
sendModificationWithChatTelemetry(insertedCode, modificationPercentage)
131146
} catch (e: Exception) {
132-
sendModificationWithChatTelemetry(insertedCode, 1.0)
147+
sendModificationWithChatTelemetry(insertedCode, null)
133148
}
134149
}
135150

136151
private fun emitTelemetryOnSuggestion(acceptedSuggestion: AcceptedSuggestionEntry) {
137152
val file = acceptedSuggestion.vFile
138153

139154
if (file == null || (!file.isValid)) {
140-
sendModificationTelemetry(acceptedSuggestion, 1.0)
155+
sendModificationTelemetry(acceptedSuggestion, null)
141156
// temp remove event sent as further discussion needed for metric calculation
142157
// sendUserModificationTelemetryToServiceAPI(acceptedSuggestion, 1.0)
143158
} else {
@@ -156,7 +171,7 @@ class CodeWhispererUserModificationTracker(private val project: Project) : Dispo
156171
// temp remove event sent as further discussion needed for metric calculation
157172
// sendUserModificationTelemetryToServiceAPI(acceptedSuggestion, modificationPercentage)
158173
} catch (e: Exception) {
159-
sendModificationTelemetry(acceptedSuggestion, 1.0)
174+
sendModificationTelemetry(acceptedSuggestion, null)
160175
// temp remove event sent as further discussion needed for metric calculation
161176
// sendUserModificationTelemetryToServiceAPI(acceptedSuggestion, 1.0)
162177
}
@@ -168,41 +183,44 @@ class CodeWhispererUserModificationTracker(private val project: Project) : Dispo
168183
* Levenshtein distance was preferred over Jaro–Winkler distance for simplicity
169184
*/
170185
@VisibleForTesting
171-
internal fun checkDiff(currString: String?, acceptedString: String?): Double {
186+
internal fun checkDiff(currString: String?, acceptedString: String?): CodeInsertionDiff? {
172187
if (currString == null || acceptedString == null || acceptedString.isEmpty() || currString.isEmpty()) {
173-
return 1.0
188+
return null
174189
}
175-
176190
val diff = checker.distance(currString, acceptedString)
177-
val percentage = diff / acceptedString.length
178-
179-
return min(1.0, percentage)
191+
return CodeInsertionDiff(
192+
original = acceptedString,
193+
modified = currString,
194+
diff = diff
195+
)
180196
}
181197

182-
private fun sendModificationTelemetry(suggestion: AcceptedSuggestionEntry, percentage: Double) {
198+
private fun sendModificationTelemetry(suggestion: AcceptedSuggestionEntry, diff: CodeInsertionDiff?) {
183199
LOG.debug { "Sending user modification telemetry. Request Id: ${suggestion.requestId}" }
184200
val startUrl = getConnectionStartUrl(suggestion.connection)
185201
CodewhispererTelemetry.userModification(
186202
project = project,
187203
codewhispererCompletionType = suggestion.completionType,
188204
codewhispererLanguage = suggestion.codewhispererLanguage.toTelemetryType(),
189-
codewhispererModificationPercentage = percentage,
205+
codewhispererModificationPercentage = diff.percentage(),
190206
codewhispererRequestId = suggestion.requestId,
191207
codewhispererRuntime = suggestion.codewhispererRuntime,
192208
codewhispererRuntimeSource = suggestion.codewhispererRuntimeSource,
193209
codewhispererSessionId = suggestion.sessionId,
194210
codewhispererSuggestionIndex = suggestion.index,
195211
codewhispererTriggerType = suggestion.triggerType,
196212
credentialStartUrl = startUrl,
197-
codewhispererUserGroup = CodeWhispererUserGroupSettings.getInstance().getUserGroup().name
213+
codewhispererUserGroup = CodeWhispererUserGroupSettings.getInstance().getUserGroup().name,
214+
codewhispererCharactersModified = diff?.modified?.length ?: 0,
215+
codewhispererCharactersAccepted = diff?.original?.length ?: 0
198216
)
199217
}
200218

201-
private fun sendModificationWithChatTelemetry(insertedCode: InsertedCodeModificationEntry, percentage: Double) {
219+
private fun sendModificationWithChatTelemetry(insertedCode: InsertedCodeModificationEntry, diff: CodeInsertionDiff?) {
202220
AmazonqTelemetry.modifyCode(
203221
cwsprChatConversationId = insertedCode.conversationId,
204222
cwsprChatMessageId = insertedCode.messageId,
205-
cwsprChatModificationPercentage = percentage,
223+
cwsprChatModificationPercentage = diff.percentage(),
206224
credentialStartUrl = getStartUrl(project)
207225
)
208226
val lang = insertedCode.vFile?.programmingLanguage() ?: CodeWhispererUnknownLanguage.INSTANCE
@@ -213,7 +231,7 @@ class CodeWhispererUserModificationTracker(private val project: Project) : Dispo
213231
insertedCode.conversationId,
214232
insertedCode.messageId,
215233
lang,
216-
percentage,
234+
diff.percentage(),
217235
CodeWhispererSettings.getInstance().isProjectContextEnabled(),
218236
CodeWhispererModelConfigurator.getInstance().activeCustomization(project)
219237
).also {

‎plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererUserModificationTrackerTest.kt‎

Lines changed: 71 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ import com.intellij.openapi.vfs.VirtualFile
1515
import com.intellij.testFramework.ProjectExtension
1616
import com.intellij.testFramework.junit5.TestDisposable
1717
import com.intellij.testFramework.replaceService
18+
import info.debatty.java.stringsimilarity.Levenshtein
1819
import org.assertj.core.api.Assertions.assertThat
1920
import org.gradle.internal.impldep.com.amazonaws.ResponseMetadata.AWS_REQUEST_ID
2021
import org.junit.jupiter.api.BeforeEach
21-
import org.junit.jupiter.api.Disabled
2222
import org.junit.jupiter.api.Test
2323
import org.junit.jupiter.api.extension.RegisterExtension
2424
import org.mockito.kotlin.any
@@ -38,10 +38,13 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.customization.Code
3838
import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator
3939
import software.aws.toolkits.jetbrains.services.codewhisperer.language.languages.CodeWhispererJava
4040
import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhispererSettings
41+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeInsertionDiff
4142
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererUserModificationTracker
43+
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.percentage
4244
import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.InsertedCodeModificationEntry
4345
import software.aws.toolkits.jetbrains.services.telemetry.MockTelemetryServiceExtension
4446
import java.time.Instant
47+
import kotlin.math.min
4548
import kotlin.test.assertNotNull
4649

4750
class CodeWhispererUserModificationTrackerTest {
@@ -61,6 +64,7 @@ class CodeWhispererUserModificationTrackerTest {
6164
private lateinit var mockModelConfigurator: CodeWhispererModelConfigurator
6265
private val project: Project
6366
get() = projectExtension.project
67+
private val levenshtein = Levenshtein()
6468

6569
companion object {
6670
@JvmField
@@ -141,11 +145,12 @@ class CodeWhispererUserModificationTrackerTest {
141145
sut.enqueue(insertedCodeModificationEntry)
142146
sut.dispose()
143147

148+
val percentageChanges = sut.checkDiff("println();", "print").percentage()
144149
verify(mockClient).sendChatUserModificationTelemetry(
145150
eq(conversationId),
146151
eq(messageId),
147152
eq(CodeWhispererJava.INSTANCE),
148-
eq(sut.checkDiff("println();", "print")),
153+
eq(percentageChanges),
149154
eq(CodeWhispererSettings.getInstance().isProjectContextEnabled()),
150155
eq(mockCustomization)
151156
)
@@ -158,15 +163,75 @@ class CodeWhispererUserModificationTrackerTest {
158163
.matches({ it.metadata["cwsprChatConversationId"] == conversationId }, "cwsprChatConversationId doesn't match")
159164
.matches({ it.metadata["cwsprChatMessageId"] == messageId }, "cwsprChatMessageId doesn't match")
160165
.matches(
161-
{ it.metadata["cwsprChatModificationPercentage"] == sut.checkDiff("println();", "print").toString() },
166+
{ it.metadata["cwsprChatModificationPercentage"] == percentageChanges.toString() },
162167
"cwsprChatModificationPercentage doesn't match"
163168
)
164169
}
165170
}
166171

167-
@Disabled
168172
@Test
169-
fun checkDiff() {
170-
// TODO
173+
fun `checkDiff edge cases`() {
174+
// any empty string will return null
175+
val r1 = sut.checkDiff("", "")
176+
assertThat(r1).isNull()
177+
178+
val r2 = sut.checkDiff("foo", "")
179+
assertThat(r2).isNull()
180+
181+
val r3 = sut.checkDiff("", "foo")
182+
assertThat(r3).isNull()
183+
184+
// null will return null
185+
val r4 = sut.checkDiff(null, null)
186+
assertThat(r4).isNull()
187+
188+
val r5 = sut.checkDiff(null, "foo")
189+
assertThat(r5).isNull()
190+
191+
val r6 = sut.checkDiff("foo", null)
192+
assertThat(r6).isNull()
193+
}
194+
195+
@Test
196+
fun `checkDiff should return data having correct payload`() {
197+
val r1 = sut.checkDiff("foo", "bar")
198+
assertThat(r1).isEqualTo(
199+
CodeInsertionDiff(modified = "foo", original = "bar", diff = levenshtein.distance("foo", "bar"))
200+
)
201+
202+
val r2 = sut.checkDiff("foo", "foo")
203+
assertThat(r2).isEqualTo(
204+
CodeInsertionDiff(modified = "foo", original = "foo", diff = levenshtein.distance("foo", "foo"))
205+
)
206+
}
207+
208+
@Test
209+
fun `CodeInsertionDiff_percentage() should return correct result`() {
210+
fun assertPercentageCorrect(original: String?, modified: String?) {
211+
val diff = sut.checkDiff(currString = modified, acceptedString = original)
212+
val expectedPercentage: Double = when {
213+
original == null || modified == null -> 1.0
214+
215+
original.isEmpty() || modified.isEmpty() -> 1.0
216+
217+
else -> min(1.0, (levenshtein.distance(modified, original) / original.length))
218+
}
219+
220+
val actual = diff.percentage()
221+
222+
assertThat(actual).isEqualTo(expectedPercentage)
223+
}
224+
225+
assertPercentageCorrect(null, null)
226+
assertPercentageCorrect(null, "foo")
227+
assertPercentageCorrect("foo", null)
228+
229+
assertPercentageCorrect("", "")
230+
assertPercentageCorrect("", "foo")
231+
assertPercentageCorrect("foo", "")
232+
233+
assertPercentageCorrect("foo", "bar")
234+
assertPercentageCorrect("foo", "foo")
235+
assertPercentageCorrect("bar", "foo")
171236
}
172237
}

0 commit comments

Comments
 (0)