feat(/dev): send FeatureDevEvent telemetry event (#4602)

sannicm · web-flow · commit 24adda9e5b5c · 2024-06-25T12:40:11.000-07:00
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevConstants.kt
@@ -3,6 +3,8 @@
 
 package software.aws.toolkits.jetbrains.services.amazonqFeatureDev
 
+const val FEATURE_EVALUATION_PRODUCT_NAME = "FeatureDev"
+
 const val FEATURE_NAME = "Amazon Q Developer Agent for software development"
 
 // Max number of times a user can attempt to retry an approach request if it fails
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/clients/FeatureDevClient.kt
@@ -6,6 +6,7 @@ package software.aws.toolkits.jetbrains.services.amazonqFeatureDev.clients
 import com.intellij.openapi.components.Service
 import com.intellij.openapi.components.service
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.SystemInfo
 import kotlinx.coroutines.future.await
 import software.amazon.awssdk.services.codewhispererruntime.CodeWhispererRuntimeClient
 import software.amazon.awssdk.services.codewhispererruntime.model.ArtifactType
@@ -14,10 +15,15 @@ import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssi
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
+import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
+import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
+import software.amazon.awssdk.services.codewhispererruntime.model.SendTelemetryEventResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.StartTaskAssistCodeGenerationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.TaskAssistPlanningUploadContext
 import software.amazon.awssdk.services.codewhispererruntime.model.UploadContext
 import software.amazon.awssdk.services.codewhispererruntime.model.UploadIntent
+import software.amazon.awssdk.services.codewhispererruntime.model.UserContext
 import software.amazon.awssdk.services.codewhispererstreaming.CodeWhispererStreamingAsyncClient
 import software.amazon.awssdk.services.codewhispererstreaming.model.ChatMessage
 import software.amazon.awssdk.services.codewhispererstreaming.model.ChatTriggerType
@@ -35,12 +41,40 @@ import software.aws.toolkits.jetbrains.core.awsClient
 import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnectionManager
 import software.aws.toolkits.jetbrains.core.credentials.pinning.QConnection
 import software.aws.toolkits.jetbrains.services.amazonq.clients.AmazonQStreamingClient
+import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FEATURE_EVALUATION_PRODUCT_NAME
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.calculateTotalLatency
+import software.aws.toolkits.jetbrains.services.telemetry.ClientMetadata
+import software.aws.toolkits.jetbrains.settings.AwsSettings
 import java.time.Instant
 import software.amazon.awssdk.services.codewhispererruntime.model.ChatTriggerType as SyncChatTriggerType
 
 @Service(Service.Level.PROJECT)
 class FeatureDevClient(private val project: Project) {
+    fun getTelemetryOptOutPreference() =
+        if (AwsSettings.getInstance().isTelemetryEnabled) {
+            OptOutPreference.OPTIN
+        } else {
+            OptOutPreference.OPTOUT
+        }
+
+    private val featureDevUserContext = ClientMetadata.getDefault().let {
+        val osForFeatureDev: OperatingSystem =
+            when {
+                SystemInfo.isWindows -> OperatingSystem.WINDOWS
+                SystemInfo.isMac -> OperatingSystem.MAC
+                // For now, categorize everything else as "Linux" (Linux/FreeBSD/Solaris/etc)
+                else -> OperatingSystem.LINUX
+            }
+
+        UserContext.builder()
+            .ideCategory(IdeCategory.JETBRAINS)
+            .operatingSystem(osForFeatureDev)
+            .product(FEATURE_EVALUATION_PRODUCT_NAME)
+            .clientId(it.clientId)
+            .ideVersion(it.awsVersion)
+            .build()
+    }
+
     private fun connection() = ToolkitConnectionManager.getInstance(project).activeConnectionForFeature(QConnection.getInstance())
         ?: error("Attempted to use connection while one does not exist")
 
@@ -51,6 +85,16 @@ class FeatureDevClient(private val project: Project) {
     private val amazonQStreamingClient
         get() = AmazonQStreamingClient.getInstance(project)
 
+    fun sendFeatureDevTelemetryEvent(conversationId: String): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
+        requestBuilder.telemetryEvent { telemetryEventBuilder ->
+            telemetryEventBuilder.featureDevEvent {
+                it.conversationId(conversationId)
+            }
+        }
+        requestBuilder.optOutPreference(getTelemetryOptOutPreference())
+        requestBuilder.userContext(featureDevUserContext)
+    }
+
     fun createTaskAssistConversation(): CreateTaskAssistConversationResponse = bearerClient().createTaskAssistConversation(
         CreateTaskAssistConversationRequest.builder().build()
     )
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/session/Session.kt
@@ -56,8 +56,8 @@ class Session(val tabID: String, val project: Project) {
         if (!preloaderFinished) {
             setupConversation(msg)
             preloaderFinished = true
-
             messenger.sendAsyncEventProgress(tabId = this.tabID, inProgress = true)
+            featureDevService.sendFeatureDevEvent(this.conversationId)
         }
     }
 
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/util/FeatureDevService.kt
@@ -9,6 +9,7 @@ import com.intellij.openapi.project.Project
 import software.amazon.awssdk.services.codewhispererruntime.model.CodeWhispererRuntimeException
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.SendTelemetryEventResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.StartTaskAssistCodeGenerationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.ValidationException
 import software.amazon.awssdk.services.codewhispererstreaming.model.CodeWhispererStreamingException
@@ -237,4 +238,17 @@ class FeatureDevService(val proxyClient: FeatureDevClient, val project: Project)
 
         return parsedResult.code_generation_result
     }
+
+    fun sendFeatureDevEvent(conversationId: String) {
+        val sendFeatureDevTelemetryEventResponse: SendTelemetryEventResponse
+        try {
+            sendFeatureDevTelemetryEventResponse = proxyClient.sendFeatureDevTelemetryEvent(conversationId)
+            val requestId = sendFeatureDevTelemetryEventResponse.responseMetadata().requestId()
+            logger.debug {
+                "$FEATURE_NAME: succesfully sent feature dev telemetry: ConversationId: $conversationId RequestId: $requestId"
+            }
+        } catch (e: Exception) {
+            logger.warn(e) { "$FEATURE_NAME: failed to send feature dev telemetry" }
+        }
+    }
 }
diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevTestBase.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevTestBase.kt
@@ -24,6 +24,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.CodeGeneration
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.SendTelemetryEventResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.StartTaskAssistCodeGenerationResponse
 import software.aws.toolkits.core.TokenConnectionSettings
 import software.aws.toolkits.core.credentials.ToolkitBearerTokenProvider
@@ -70,6 +71,10 @@ open class FeatureDevTestBase(
     internal val testChecksumSha = "test-sha"
     internal val testContentLength: Long = 40
 
+    internal val exampleSendTelemetryEventResponse = SendTelemetryEventResponse.builder()
+        .responseMetadata(DefaultAwsResponseMetadata.create(mapOf(AwsHeader.AWS_REQUEST_ID to testRequestId)))
+        .build() as SendTelemetryEventResponse
+
     internal val exampleCreateTaskAssistConversationResponse = CreateTaskAssistConversationResponse.builder()
         .conversationId(testConversationId)
         .responseMetadata(DefaultAwsResponseMetadata.create(mapOf(AwsHeader.AWS_REQUEST_ID to testRequestId)))
diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevControllerTest.kt
@@ -143,6 +143,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         doReturn(testAuth).`when`(authController).getAuthNeededStates(any())
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(featureDevClient.createTaskAssistUploadUrl(any(), any(), any())).thenReturn(exampleCreateUploadUrlResponse)
 
         runTest {
@@ -168,6 +169,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         val message = IncomingFeatureDevMessage.FollowupClicked(followUp, testTabId, "", "test-command")
 
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
         doNothing().`when`(chatSessionStorage).deleteSession(any())
 
@@ -216,6 +218,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         val message = IncomingFeatureDevMessage.FollowupClicked(followUp, testTabId, "", "test-command")
 
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
 
         mockkObject(AmazonqTelemetry)
@@ -243,6 +246,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         } just runs
 
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
         whenever(spySession.sessionState).thenReturn(
             PrepareCodeGenerationState(
@@ -403,6 +407,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         val message = IncomingFeatureDevMessage.FollowupClicked(followUp, testTabId, "", "test-command")
 
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
 
         mockkStatic("software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.FileUtilsKt")
@@ -431,6 +436,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         val message = IncomingFeatureDevMessage.FollowupClicked(followUp, testTabId, "", "test-command")
 
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
 
         mockkStatic("software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.FileUtilsKt")
@@ -464,6 +470,7 @@ class FeatureDevControllerTest : FeatureDevTestBase() {
         val message = IncomingFeatureDevMessage.FollowupClicked(followUp, testTabId, "", "test-command")
 
         whenever(featureDevClient.createTaskAssistConversation()).thenReturn(exampleCreateTaskAssistConversationResponse)
+        whenever(featureDevClient.sendFeatureDevTelemetryEvent(any())).thenReturn(exampleSendTelemetryEventResponse)
         whenever(chatSessionStorage.getSession(any(), any())).thenReturn(spySession)
 
         val folder = LightVirtualFile("${spySession.context.projectRoot.name}/path/to/sub/folder")
diff --git a/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json b/plugins/core/sdk-codegen/codegen-resources/codewhispererruntime/service-2.json
@@ -456,6 +456,13 @@
                 "DOWNVOTE"
             ]
         },
+        "FeatureDevEvent": {
+            "type": "structure",
+            "required": ["conversationId"],
+            "members": {
+                "conversationId": { "shape":  "ConversationId" }
+            }
+        },
         "ChatUserModificationEvent": {
             "type": "structure",
             "required": ["conversationId", "messageId", "modificationPercentage"],
@@ -1593,7 +1600,8 @@
                 "metricData":{"shape":"MetricData"},
                 "chatAddMessageEvent": { "shape": "ChatAddMessageEvent" },
                 "chatInteractWithMessageEvent": { "shape": "ChatInteractWithMessageEvent" },
-                "chatUserModificationEvent": { "shape": "ChatUserModificationEvent" }
+                "chatUserModificationEvent": { "shape": "ChatUserModificationEvent" },
+                "featureDevEvent":  { "shape":  "FeatureDevEvent" }
             },
             "union":true
         },

Original file line number	Diff line number	Diff line change
`@@ -56,8 +56,8 @@ class Session(val tabID: String, val project: Project) {`
`56`	`56`	`if (!preloaderFinished) {`
`57`	`57`	`setupConversation(msg)`
`58`	`58`	`preloaderFinished = true`
`59`		`-`
`60`	`59`	`messenger.sendAsyncEventProgress(tabId = this.tabID, inProgress = true)`
	`60`	`+ featureDevService.sendFeatureDevEvent(this.conversationId)`
`61`	`61`	`}`
`62`	`62`	`}`
`63`	`63`