Skip to content

Commit 662a251

Browse files
dhasani23David Hasani
dhasani23
and
David Hasani
authored
telemetry(amazonq): emit TransformEvent metric from download ZIP (#5038)
* telemetry(amazonq): emit TransformEvent metric from download ZIP * add LOC submitted data point * emit metric only after initial download * fix test * handle case where metrics.json is missing * add try-catch * update api model * remove accidentally-committed file * remove notifys * remove unused import * fix unit test * address rli comments --------- Co-authored-by: David Hasani <[email protected]>
1 parent c8d82c3 commit 662a251

File tree

11 files changed

+132
-28
lines changed

11 files changed

+132
-28
lines changed

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/ArtifactHandler.kt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.model.DownloadFai
3939
import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
4040
import software.aws.toolkits.jetbrains.services.codemodernizer.model.ParseZipFailureReason
4141
import software.aws.toolkits.jetbrains.services.codemodernizer.model.UnzipFailureReason
42+
import software.aws.toolkits.jetbrains.services.codemodernizer.state.CodeModernizerSessionState
4243
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getPathToHilArtifactDir
4344
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.isValidCodeTransformConnection
4445
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.openTroubleshootingGuideNotificationAction
@@ -199,6 +200,17 @@ class ArtifactHandler(private val project: Project, private val clientAdaptor: G
199200
downloadedBuildLogPath[job] = path
200201
} else {
201202
downloadedArtifacts[job] = path
203+
if (output.artifact is CodeModernizerArtifact && output.artifact.metrics != null) {
204+
output.artifact.metrics.linesOfCodeSubmitted = CodeModernizerSessionState.getInstance(project).getLinesOfCodeSubmitted()
205+
output.artifact.metrics.programmingLanguage = CodeModernizerSessionState.getInstance(project).getTransformationLanguage()
206+
try {
207+
clientAdaptor.sendTransformTelemetryEvent(job, output.artifact.metrics)
208+
} catch (e: Exception) {
209+
// log error, but can still show diff.patch and summary.md
210+
LOG.error { e.message.toString() }
211+
telemetryErrorMessage = "Unexpected error when sending telemetry with metrics ${e.localizedMessage}"
212+
}
213+
}
202214
}
203215
output
204216
} catch (e: RuntimeException) {

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformat
1616
import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformationPlanResponse
1717
import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformationRequest
1818
import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformationResponse
19+
import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
1920
import software.amazon.awssdk.services.codewhispererruntime.model.ResumeTransformationRequest
2021
import software.amazon.awssdk.services.codewhispererruntime.model.ResumeTransformationResponse
2122
import software.amazon.awssdk.services.codewhispererruntime.model.StartTransformationRequest
@@ -46,8 +47,11 @@ import software.aws.toolkits.jetbrains.services.amazonq.CONTENT_SHA256
4647
import software.aws.toolkits.jetbrains.services.amazonq.SERVER_SIDE_ENCRYPTION
4748
import software.aws.toolkits.jetbrains.services.amazonq.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID
4849
import software.aws.toolkits.jetbrains.services.amazonq.clients.AmazonQStreamingClient
50+
import software.aws.toolkits.jetbrains.services.amazonq.codeWhispererUserContext
51+
import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerMetrics
4952
import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
5053
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.calculateTotalLatency
54+
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
5155
import java.io.File
5256
import java.net.HttpURLConnection
5357
import java.time.Instant
@@ -210,6 +214,26 @@ class GumbyClient(private val project: Project) {
210214
}
211215
}
212216

217+
fun sendTransformTelemetryEvent(job: JobId, metrics: CodeModernizerMetrics) {
218+
bearerClient().sendTelemetryEvent { requestBuilder ->
219+
requestBuilder.telemetryEvent { telemetryEventBuilder ->
220+
telemetryEventBuilder.transformEvent {
221+
it.jobId(job.id)
222+
it.timestamp(Instant.now())
223+
it.ideCategory(IdeCategory.JETBRAINS)
224+
it.programmingLanguage { language ->
225+
language.languageName(metrics.programmingLanguage)
226+
}
227+
it.linesOfCodeChanged(metrics.linesOfCodeChanged)
228+
it.charsOfCodeChanged(metrics.charactersOfCodeChanged)
229+
it.linesOfCodeSubmitted(metrics.linesOfCodeSubmitted) // currently unavailable for SQL conversions
230+
}
231+
}
232+
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
233+
requestBuilder.userContext(codeWhispererUserContext())
234+
}
235+
}
236+
213237
companion object {
214238
private val LOG = getLogger<GumbyClient>()
215239

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerArtifact.kt‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package software.aws.toolkits.jetbrains.services.codemodernizer.model
55

66
import com.fasterxml.jackson.core.JsonProcessingException
77
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
8+
import com.fasterxml.jackson.module.kotlin.readValue
89
import com.intellij.openapi.util.io.FileUtil.createTempDirectory
910
import com.intellij.openapi.vfs.LocalFileSystem
1011
import com.intellij.openapi.vfs.VirtualFile
@@ -29,17 +30,19 @@ open class CodeModernizerArtifact(
2930
private val patches: List<VirtualFile>,
3031
val summary: TransformationSummary,
3132
val summaryMarkdownFile: File,
33+
val metrics: CodeModernizerMetrics?,
3234
) : CodeTransformDownloadArtifact {
3335
val patch: VirtualFile
3436
get() = patches.first()
3537

3638
companion object {
37-
private const val maxSupportedVersion = 1.0
39+
private const val MAX_SUPPORTED_VERSION = 1.0
3840
private val tempDir = createTempDirectory("codeTransformArtifacts", null)
39-
private const val manifestPathInZip = "manifest.json"
40-
private const val summaryNameInZip = "summary.md"
41+
private const val MANIFEST_FILE_NAME = "manifest.json"
42+
private const val SUMMARY_FILE_NAME = "summary.md"
43+
private const val METRICS_FILE_NAME = "metrics.json"
4144
val LOG = getLogger<CodeModernizerArtifact>()
42-
private val MAPPER = jacksonObjectMapper()
45+
val MAPPER = jacksonObjectMapper()
4346

4447
/**
4548
* Extracts the file at [zipPath] and uses its contents to produce a [CodeModernizerArtifact].
@@ -53,39 +56,40 @@ open class CodeModernizerArtifact(
5356
throw RuntimeException("Could not unzip artifact")
5457
}
5558
val manifest = loadManifest()
56-
if (manifest.version > maxSupportedVersion) {
59+
if (manifest.version > MAX_SUPPORTED_VERSION) {
5760
// If not supported we can still try to use it, i.e. the versions should largely be backwards compatible
58-
LOG.warn { "Unsupported version: ${manifest.version}" }
61+
LOG.warn { "Unsupported manifest.json version: ${manifest.version}" }
5962
}
6063
val patches = extractPatches(manifest)
6164
val summary = extractSummary(manifest)
6265
val summaryMarkdownFile = getSummaryFile(manifest)
66+
val metrics = loadMetrics(manifest)
6367
if (patches.size != 1) throw RuntimeException("Expected 1 patch, but found ${patches.size}")
64-
return CodeModernizerArtifact(zipPath, manifest, patches, summary, summaryMarkdownFile)
68+
return CodeModernizerArtifact(zipPath, manifest, patches, summary, summaryMarkdownFile, metrics)
6569
}
6670
throw RuntimeException("Could not find artifact")
6771
}
6872

6973
private fun extractSummary(manifest: CodeModernizerManifest): TransformationSummary {
70-
val summaryFile = tempDir.toPath().resolve(manifest.summaryRoot).resolve(summaryNameInZip).toFile()
74+
val summaryFile = tempDir.toPath().resolve(manifest.summaryRoot).resolve(SUMMARY_FILE_NAME).toFile()
7175
if (!summaryFile.exists() || summaryFile.isDirectory) {
7276
throw RuntimeException("The summary in the downloaded zip had an unknown format")
7377
}
7478
return TransformationSummary(summaryFile.readText())
7579
}
7680

77-
private fun getSummaryFile(manifest: CodeModernizerManifest) = tempDir.toPath().resolve(manifest.summaryRoot).resolve(summaryNameInZip).toFile()
81+
private fun getSummaryFile(manifest: CodeModernizerManifest) = tempDir.toPath().resolve(manifest.summaryRoot).resolve(SUMMARY_FILE_NAME).toFile()
7882

7983
/**
8084
* Attempts to load the manifest from the zip file. Throws an exception if the manifest is not found or cannot be serialized.
8185
*/
8286
private fun loadManifest(): CodeModernizerManifest {
8387
val manifestFile =
8488
tempDir.listFiles()
85-
?.firstOrNull { it.name.endsWith(manifestPathInZip) }
89+
?.firstOrNull { it.name.endsWith(MANIFEST_FILE_NAME) }
8690
?: throw RuntimeException("Could not find manifest")
8791
try {
88-
val manifest = MAPPER.readValue(manifestFile, CodeModernizerManifest::class.java)
92+
val manifest = MAPPER.readValue<CodeModernizerManifest>(manifestFile)
8993
if (manifest.version == 0.0F) {
9094
throw RuntimeException(
9195
"Unable to deserialize the manifest",
@@ -97,6 +101,19 @@ open class CodeModernizerArtifact(
97101
}
98102
}
99103

104+
private fun loadMetrics(manifest: CodeModernizerManifest): CodeModernizerMetrics? {
105+
try {
106+
val metricsFile =
107+
tempDir.resolve(manifest.metricsRoot).listFiles()
108+
?.firstOrNull { it.name.endsWith(METRICS_FILE_NAME) }
109+
?: throw RuntimeException("Could not find metrics.json")
110+
return MAPPER.readValue<CodeModernizerMetrics>(metricsFile)
111+
} catch (exception: Exception) {
112+
// if metrics.json not present or parsing fails, can still show diff.patch and summary.md
113+
return null
114+
}
115+
}
116+
100117
@OptIn(ExperimentalPathApi::class)
101118
private fun extractPatches(manifest: CodeModernizerManifest): List<VirtualFile> {
102119
val fileSystem = LocalFileSystem.getInstance()

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerManifest.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ package software.aws.toolkits.jetbrains.services.codemodernizer.model
66
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
77

88
@JsonIgnoreProperties(ignoreUnknown = true)
9-
data class CodeModernizerManifest(val version: Float, val patchesRoot: String, val artifactsRoot: String, val summaryRoot: String)
9+
data class CodeModernizerManifest(val version: Float, val patchesRoot: String, val artifactsRoot: String, val summaryRoot: String, val metricsRoot: String)

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerMetrics.kt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package software.aws.toolkits.jetbrains.services.codemodernizer.model
5+
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
7+
8+
@JsonIgnoreProperties(ignoreUnknown = true)
9+
data class CodeModernizerMetrics(
10+
val linesOfCodeChanged: Int?,
11+
val charactersOfCodeChanged: Int?,
12+
var linesOfCodeSubmitted: Int?,
13+
var programmingLanguage: String?,
14+
)

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/plan/CodeModernizerPlanEditor.kt‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33

44
package software.aws.toolkits.jetbrains.services.codemodernizer.plan
55

6-
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
76
import com.fasterxml.jackson.module.kotlin.readValue
87
import com.intellij.ide.BrowserUtil
98
import com.intellij.openapi.fileEditor.FileEditor
@@ -22,11 +21,14 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.constants.APPENDI
2221
import software.aws.toolkits.jetbrains.services.codemodernizer.constants.CodeModernizerUIConstants
2322
import software.aws.toolkits.jetbrains.services.codemodernizer.constants.JOB_STATISTICS_TABLE_KEY
2423
import software.aws.toolkits.jetbrains.services.codemodernizer.constants.LOC_THRESHOLD
24+
import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerArtifact.Companion.MAPPER
2525
import software.aws.toolkits.jetbrains.services.codemodernizer.model.PlanTable
2626
import software.aws.toolkits.jetbrains.services.codemodernizer.plan.CodeModernizerPlanEditorProvider.Companion.MIGRATION_PLAN_KEY
2727
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getAuthType
2828
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getBillingText
29+
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getLinesOfCodeSubmitted
2930
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getTableMapping
31+
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.parseTableMapping
3032
import software.aws.toolkits.jetbrains.services.codewhisperer.layout.CodeWhispererLayoutConfig.addHorizontalGlue
3133
import software.aws.toolkits.resources.message
3234
import software.aws.toolkits.telemetry.CredentialSourceId
@@ -59,13 +61,7 @@ import javax.swing.table.DefaultTableModel
5961

6062
class CodeModernizerPlanEditor(val project: Project, private val virtualFile: VirtualFile) : UserDataHolderBase(), FileEditor {
6163
val plan = virtualFile.getUserData(MIGRATION_PLAN_KEY) ?: throw RuntimeException("Migration plan not found")
62-
private val tableMapping =
63-
if (!plan.transformationSteps()[0].progressUpdates().isNullOrEmpty()) {
64-
getTableMapping(plan.transformationSteps()[0].progressUpdates())
65-
} else {
66-
throw RuntimeException("GetPlan response missing step 0 progress updates with table data")
67-
}
68-
private val mapper = jacksonObjectMapper()
64+
private val tableMapping = getTableMapping(plan.transformationSteps()[0].progressUpdates())
6965

7066
// to-do: convert to UI DSL
7167
private val contentPanel =
@@ -79,8 +75,8 @@ class CodeModernizerPlanEditor(val project: Project, private val virtualFile: Vi
7975
// key "0" reserved for job statistics table
8076
// comes from "name" field of each progressUpdate in step zero of plan
8177
if (JOB_STATISTICS_TABLE_KEY in tableMapping) {
82-
val planTable = mapper.readValue(tableMapping[JOB_STATISTICS_TABLE_KEY], PlanTable::class.java)
83-
val linesOfCode = planTable.rows.find { it.name == "linesOfCode" }?.value?.toInt()
78+
val planTable = parseTableMapping(tableMapping)
79+
val linesOfCode = planTable?.let { getLinesOfCodeSubmitted(it) }
8480
if (linesOfCode != null && linesOfCode > LOC_THRESHOLD && getAuthType(project) == CredentialSourceId.IamIdentityCenter) {
8581
val billingText = getBillingText(linesOfCode)
8682
val billingTextComponent =
@@ -102,15 +98,15 @@ class CodeModernizerPlanEditor(val project: Project, private val virtualFile: Vi
10298
add(billingTextComponent, CodeModernizerUIConstants.transformationPlanPlaneConstraint)
10399
}
104100
add(
105-
transformationPlanInfo(planTable),
101+
planTable?.let { transformationPlanInfo(it) },
106102
CodeModernizerUIConstants.transformationPlanPlaneConstraint,
107103
)
108104
}
109105
add(transformationPlanPanel(plan), CodeModernizerUIConstants.transformationPlanPlaneConstraint)
110106
// key "-1" reserved for appendix table
111107
if (APPENDIX_TABLE_KEY in tableMapping) {
112108
add(
113-
transformationPlanAppendix(mapper.readValue(tableMapping[APPENDIX_TABLE_KEY], PlanTable::class.java)),
109+
tableMapping[APPENDIX_TABLE_KEY]?.let { MAPPER.readValue<PlanTable>(it) }?.let { transformationPlanAppendix(it) },
114110
CodeModernizerUIConstants.transformationPlanPlaneConstraint,
115111
)
116112
}
@@ -400,7 +396,7 @@ class CodeModernizerPlanEditor(val project: Project, private val virtualFile: Vi
400396
val table = tableMapping[step.id()]
401397

402398
val parsedTable = table?.let {
403-
mapper.readValue<PlanTable>(it)
399+
MAPPER.readValue<PlanTable>(it)
404400
}
405401

406402
val renderedStepTable = parsedTable?.let {

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/state/CodeModernizerSessionState.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModerni
1212
import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerSessionContext
1313
import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobHistoryItem
1414
import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
15+
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getLinesOfCodeSubmitted
16+
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getTableMapping
17+
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.parseTableMapping
1518
import java.time.Duration
1619
import java.time.Instant
1720
import kotlin.io.path.Path
@@ -55,4 +58,14 @@ class CodeModernizerSessionState {
5558
}
5659

5760
fun getJobHistory(): Array<JobHistoryItem> = previousJobHistory.values.toTypedArray()
61+
62+
// LOC submitted only available for Java upgrades
63+
fun getLinesOfCodeSubmitted(): Int? {
64+
val tableMapping = transformationPlan?.transformationSteps()?.get(0)?.let { getTableMapping(it.progressUpdates()) }
65+
val planTable = tableMapping?.let { parseTableMapping(it) }
66+
return planTable?.let { getLinesOfCodeSubmitted(it) }
67+
}
68+
69+
// we only create a transformationPlan for Java upgrades
70+
fun getTransformationLanguage(): String = if (transformationPlan != null) "JAVA" else "SQL"
5871
}

‎plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/utils/CodeTransformApiUtils.kt‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
package software.aws.toolkits.jetbrains.services.codemodernizer.utils
55

6+
import com.fasterxml.jackson.module.kotlin.readValue
67
import com.intellij.openapi.project.Project
78
import com.intellij.serviceContainer.AlreadyDisposedException
89
import kotlinx.coroutines.delay
@@ -23,7 +24,10 @@ import software.aws.toolkits.core.utils.Waiters.waitUntil
2324
import software.aws.toolkits.jetbrains.services.codemodernizer.CodeTransformTelemetryManager
2425
import software.aws.toolkits.jetbrains.services.codemodernizer.client.GumbyClient
2526
import software.aws.toolkits.jetbrains.services.codemodernizer.constants.BILLING_RATE
27+
import software.aws.toolkits.jetbrains.services.codemodernizer.constants.JOB_STATISTICS_TABLE_KEY
28+
import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerArtifact.Companion.MAPPER
2629
import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
30+
import software.aws.toolkits.jetbrains.services.codemodernizer.model.PlanTable
2731
import software.aws.toolkits.resources.message
2832
import java.time.Duration
2933
import java.util.Locale
@@ -128,11 +132,23 @@ suspend fun JobId.pollTransformationStatusAndPlan(
128132
}
129133

130134
// "name" holds the ID of the corresponding plan step (where table will go) and "description" holds the plan data
131-
fun getTableMapping(stepZeroProgressUpdates: List<TransformationProgressUpdate>) = stepZeroProgressUpdates.associate {
132-
it.name() to it.description()
135+
fun getTableMapping(stepZeroProgressUpdates: List<TransformationProgressUpdate>): Map<String, String> {
136+
if (stepZeroProgressUpdates.isNotEmpty()) {
137+
return stepZeroProgressUpdates.associate {
138+
it.name() to it.description()
139+
}
140+
} else {
141+
error("GetPlan response missing step 0 progress updates with table data")
142+
}
133143
}
134144

145+
fun parseTableMapping(tableMapping: Map<String, String>): PlanTable? =
146+
tableMapping[JOB_STATISTICS_TABLE_KEY]?.let { MAPPER.readValue<PlanTable>(it) }
147+
135148
fun getBillingText(linesOfCode: Int): String {
136149
val estimatedCost = String.format(Locale.US, "%.2f", linesOfCode.times(BILLING_RATE))
137150
return message("codemodernizer.migration_plan.header.billing_text", linesOfCode, BILLING_RATE, estimatedCost)
138151
}
152+
153+
fun getLinesOfCodeSubmitted(planTable: PlanTable) =
154+
planTable.rows.find { it.name == "linesOfCode" }?.value?.toInt()

‎plugins/amazonq/codetransform/jetbrains-community/tst-resources/codemodernizer/simple.zip‎

22.1 KB
Binary file not shown.

0 commit comments

Comments
 (0)