fix(appcomposer): add main Application Composer telemetry (#1559)

JLargent42 · Jacob Largent · web-flow · commit e6e38a585e5e · 2023-11-29T13:26:42.000-08:00
## Problem:
Application Composer does not have much telemetry, so we are
not able to easily determine potential problems or pain points.

## Solution:
This adds telemetry events for load time, FPS, add resource,
add connection, open/close WFS, and generate failed.

---------

Co-authored-by: Jacob Largent &lt;largjaco@amazon.com&gt;
diff --git a/src/applicationcomposer/messageHandlers/deployMessageHandler.ts b/src/applicationcomposer/messageHandlers/deployMessageHandler.ts
@@ -10,7 +10,9 @@ import { telemetry } from '../../shared/telemetry/telemetry'
 
 export async function deployMessageHandler(message: DeployRequestMessage, context: WebviewContext) {
     // SAM already handles success/failure, so we only log that the user clicked the deploy button
-    telemetry.appcomposer_deployClicked.emit()
+    telemetry.appcomposer_deployClicked.emit({
+        result: 'Succeeded',
+    })
     const args = context.textDocument.uri
     /* TODO Rework this command so that a failure case is returned
      * We don't want to override the SAM Sync telemetry. The SAM telemetry catches all errors,
diff --git a/src/applicationcomposer/messageHandlers/emitTelemetryMessageHandler.ts b/src/applicationcomposer/messageHandlers/emitTelemetryMessageHandler.ts
@@ -4,33 +4,157 @@
  */
 
 import { EmitTelemetryMessage } from '../types'
-import { telemetry } from '../../shared/telemetry/telemetry'
+import {
+    AppcomposerAddConnection,
+    AppcomposerAddResource,
+    AppcomposerCloseWfs,
+    AppcomposerCustomerReady,
+    AppcomposerFps,
+    AppcomposerGenerateAccepted,
+    AppcomposerGenerateClicked,
+    AppcomposerGenerateRejected,
+    AppcomposerInvalidGeneration,
+    AppcomposerOpenWfs,
+    AppcomposerPostProcess,
+    AppcomposerRegenerateClicked,
+    telemetry,
+} from '../../shared/telemetry/telemetry'
+import { getLogger } from '../../shared/logger'
 
 export function emitTelemetryMessageHandler(message: EmitTelemetryMessage) {
-    switch (message.eventType) {
-        case 'GENERATE_CLICKED':
-            telemetry.appcomposer_generateClicked.emit({
-                result: 'Succeeded',
-                resourceType: message.resourceId,
-            })
-            return
-        case 'REGENERATE_CLICKED':
-            telemetry.appcomposer_regenerateClicked.emit({
-                result: 'Succeeded',
-                resourceType: message.resourceId,
-            })
-            return
-        case 'GENERATE_ACCEPTED':
-            telemetry.appcomposer_generateAccepted.emit({
-                result: 'Succeeded',
-                resourceType: message.resourceId,
-            })
-            return
-        case 'GENERATE_REJECTED':
-            telemetry.appcomposer_generateRejected.emit({
-                result: 'Succeeded',
-                resourceType: message.resourceId,
-            })
-            return
+    try {
+        const parsedData = message.metadata ? JSON.parse(message.metadata) : {}
+        switch (message.eventType) {
+            case 'GENERATE_CLICKED':
+                sendGenerateClicked(parsedData as AppcomposerGenerateClicked)
+                return
+            case 'REGENERATE_CLICKED':
+                sendRegenerateClicked(parsedData as AppcomposerRegenerateClicked)
+                return
+            case 'GENERATE_ACCEPTED':
+                sendGenerateAccepted(parsedData as AppcomposerGenerateAccepted)
+                return
+            case 'GENERATE_REJECTED':
+                sendGenerateRejected(parsedData as AppcomposerGenerateRejected)
+                return
+            case 'INVALID_GENERATION':
+                sendInvalidGeneration(parsedData as AppcomposerInvalidGeneration)
+                return
+            case 'POST_PROCESS':
+                sendPostProcess(parsedData as AppcomposerPostProcess)
+                return
+            case 'CUSTOMER_READY':
+                sendCustomerReady(parsedData as AppcomposerCustomerReady)
+                return
+            case 'FPS':
+                sendFps(parsedData as AppcomposerFps)
+                return
+            case 'ADD_RESOURCE':
+                sendAddResource(parsedData as AppcomposerAddResource)
+                return
+            case 'ADD_CONNECTION':
+                sendAddConnection(parsedData as AppcomposerAddConnection)
+                return
+            case 'OPEN_WFS':
+                sendOpenWfs(parsedData as AppcomposerOpenWfs)
+                return
+            case 'CLOSE_WFS':
+                sendCloseWfs(parsedData as AppcomposerCloseWfs)
+                return
+        }
+    } catch (e) {
+        getLogger().error('Could not log telemetry for App Composer', e)
     }
 }
+
+function sendGenerateClicked(metadata: AppcomposerGenerateClicked) {
+    telemetry.appcomposer_generateClicked.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+    })
+}
+
+function sendRegenerateClicked(metadata: AppcomposerRegenerateClicked) {
+    telemetry.appcomposer_regenerateClicked.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+    })
+}
+
+function sendGenerateAccepted(metadata: AppcomposerGenerateAccepted) {
+    telemetry.appcomposer_generateAccepted.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+        numAttempts: metadata.numAttempts,
+    })
+}
+
+function sendGenerateRejected(metadata: AppcomposerGenerateRejected) {
+    telemetry.appcomposer_generateRejected.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+        numAttempts: metadata.numAttempts,
+    })
+}
+
+function sendInvalidGeneration(metadata: AppcomposerInvalidGeneration) {
+    telemetry.appcomposer_invalidGeneration.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+        generateFailure: metadata.generateFailure,
+    })
+}
+
+function sendPostProcess(metadata: AppcomposerPostProcess) {
+    telemetry.appcomposer_postProcess.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+        pathsScrubbed: metadata.pathsScrubbed,
+    })
+}
+
+function sendCustomerReady(metadata: AppcomposerCustomerReady) {
+    telemetry.appcomposer_customerReady.emit({
+        result: metadata.result ?? 'Succeeded',
+        loadFileTime: metadata.loadFileTime,
+        initializeTime: metadata.initializeTime,
+        saveFileTime: metadata.saveFileTime,
+    })
+}
+
+function sendFps(metadata: AppcomposerFps) {
+    telemetry.appcomposer_fps.emit({
+        result: metadata.result ?? 'Succeeded',
+        fps: metadata.fps,
+    })
+}
+
+function sendAddResource(metadata: AppcomposerAddResource) {
+    telemetry.appcomposer_addResource.emit({
+        result: metadata.result ?? 'Succeeded',
+        resourceType: metadata.resourceType,
+    })
+}
+
+function sendAddConnection(metadata: AppcomposerAddConnection) {
+    telemetry.appcomposer_addConnection.emit({
+        result: metadata.result ?? 'Succeeded',
+        sourceResourceType: metadata.sourceResourceType,
+        sourceFacetType: metadata.sourceFacetType,
+        destResourceType: metadata.destResourceType,
+        destFacetType: metadata.destFacetType,
+    })
+}
+
+function sendOpenWfs(metadata: AppcomposerOpenWfs) {
+    telemetry.appcomposer_openWfs.emit({
+        result: metadata.result ?? 'Succeeded',
+    })
+}
+
+function sendCloseWfs(metadata: AppcomposerCloseWfs) {
+    telemetry.appcomposer_closeWfs.emit({
+        result: metadata.result ?? 'Succeeded',
+        didSave: metadata.didSave,
+    })
+}
diff --git a/src/applicationcomposer/types.ts b/src/applicationcomposer/types.ts
@@ -42,6 +42,21 @@ export enum MessageType {
     BROADCAST = 'BROADCAST',
 }
 
+enum TelemetryType {
+    GENERATE_CLICKED = 'GENERATE_CLICKED',
+    REGENERATE_CLICKED = 'REGENERATE_CLICKED',
+    GENERATE_ACCEPTED = 'GENERATE_ACCEPTED',
+    GENERATE_REJECTED = 'GENERATE_REJECTED',
+    INVALID_GENERATION = 'INVALID_GENERATION',
+    POST_PROCESS = 'POST_PROCESS',
+    CUSTOMER_READY = 'CUSTOMER_READY',
+    FPS = 'FPS',
+    ADD_RESOURCE = 'ADD_RESOURCE',
+    ADD_CONNECTION = 'ADD_CONNECTION',
+    OPEN_WFS = 'OPEN_WFS',
+    CLOSE_WFS = 'CLOSE_WFS',
+}
+
 export interface InitResponseMessage extends Message {
     templateFileName: string
     templateFilePath: string
@@ -121,8 +136,8 @@ export interface LogMessage extends Message {
 
 export interface EmitTelemetryMessage extends Message {
     eventId?: string
-    eventType: 'GENERATE_CLICKED' | 'REGENERATE_CLICKED' | 'GENERATE_ACCEPTED' | 'GENERATE_REJECTED'
-    resourceId: string
+    eventType: TelemetryType
+    metadata?: string
 }
 
 export interface ReferenceDetails {
diff --git a/src/shared/telemetry/vscodeTelemetry.json b/src/shared/telemetry/vscodeTelemetry.json
@@ -58,11 +58,72 @@
             "type": "boolean",
             "description": "Whether the user has access to CodeWhisperer Chat"
         },
+        {
+            "name": "numAttempts",
+            "type": "int",
+            "description": "Number of generations before the user accepted or rejected"
+        },
         {
             "name": "resourceType",
             "type": "string",
             "description": "CloudFormation resource type"
         },
+        {
+            "name": "pathsScrubbed",
+            "type": "int",
+            "description": "Number of generation paths modified"
+        },
+        {
+            "name": "generateFailure",
+            "type": "string",
+            "allowedValues": ["Parse"],
+            "description": "Type of generation failure"
+        },
+        {
+            "name": "loadFileTime",
+            "type": "int",
+            "description": "Time from opening Composer to loading the file"
+        },
+        {
+            "name": "initializeTime",
+            "type": "int",
+            "description": "Time from opening Composer to Composer initialize"
+        },
+        {
+            "name": "saveFileTime",
+            "type": "int",
+            "description": "Time from opening Composer to saving the file"
+        },
+        {
+            "name": "fps",
+            "type": "int",
+            "description": "Average frames per second"
+        },
+        {
+            "name": "sourceResourceType",
+            "type": "string",
+            "description": "CloudFormation resource type of connection start"
+        },
+        {
+            "name": "destResourceType",
+            "type": "string",
+            "description": "CloudFormation resource type of connection end"
+        },
+        {
+            "name": "sourceFacetType",
+            "type": "string",
+            "description": "Resource subtype of connection start"
+        },
+        {
+            "name": "destFacetType",
+            "type": "string",
+            "description": "Resource subtype of connection end"
+        },
+        {
+            "name": "didSave",
+            "type": "boolean",
+            "description": "Whether the user saved"
+        },
         {
             "name": "cwsprChatTriggerInteraction",
             "type": "string",
@@ -549,13 +610,57 @@
         {
             "name": "appcomposer_generateAccepted",
             "description": "Called after accepting a generative AI suggestion",
-            "metadata": [{ "type": "resourceType" }]
+            "metadata": [{ "type": "resourceType" }, { "type": "numAttempts" }]
         },
         {
             "name": "appcomposer_generateRejected",
             "description": "Called after rejecting a generative AI suggestion",
+            "metadata": [{ "type": "resourceType" }, { "type": "numAttempts" }]
+        },
+        {
+            "name": "appcomposer_invalidGeneration",
+            "description": "Called after a generate failed",
+            "metadata": [{ "type": "resourceType" }, { "type": "generateFailure" }]
+        },
+        {
+            "name": "appcomposer_postProcess",
+            "description": "Called after a generate is processed",
+            "metadata": [{ "type": "resourceType" }, { "type": "pathsScrubbed" }]
+        },
+        {
+            "name": "appcomposer_customerReady",
+            "description": "Called after a customer loads into Composeer",
+            "metadata": [{ "type": "loadFileTime" }, { "type": "initializeTime" }, { "type": "saveFileTime" }]
+        },
+        {
+            "name": "appcomposer_fps",
+            "description": "Called regularly to record app performance",
+            "metadata": [{ "type": "fps" }]
+        },
+        {
+            "name": "appcomposer_addResource",
+            "description": "Called when a resource is added to the canvas",
             "metadata": [{ "type": "resourceType" }]
         },
+        {
+            "name": "appcomposer_addConnection",
+            "description": "Called when two resources are connected",
+            "metadata": [
+                { "type": "sourceResourceType" },
+                { "type": "sourceFacetType" },
+                { "type": "destResourceType" },
+                { "type": "destFacetType" }
+            ]
+        },
+        {
+            "name": "appcomposer_openWfs",
+            "description": "Called when Step Functions Workflow Studio is opened"
+        },
+        {
+            "name": "appcomposer_closeWfs",
+            "description": "Called when Step Functions Workflow Studio is closed",
+            "metadata": [{ "type": "didSave" }]
+        },
         {
             "name": "codewhisperer_codeScanIssueHover",
             "description": "Called when a code scan issue is hovered over",
