telemetry(webview): Emit toolkit_X module telemetry on auth webview

Problem:

With our telemetry, we do not know when the frontend webview UI has actually loaded

Solution:

Emit certain metrics during the webview loading process to get a better idea of if the
webview UI successfully completed its initial load.

- toolkit_willOpenModule, indicates intent to render a webview
- toolkit_didLoadModule, indicates the final result of loading the webview
  - On Success it it just a success result. We know a success when the frontend send a successful
    message to the backend. It knows this by ensuring there were no errors and that a certain HTML element
    can be found, then once the page finishes its initial load it will send a success message to the backend.
  - On Failure, what happens is a timer times out after 10 seconds.

Signed-off-by: nkomonen-amazon <[email protected]>

Author: nkomonen-amazon

packages/core/src/amazonq/webview/messages/messageDispatcher.ts

@@ -16,6 +16,8 @@ import globals from '../../../shared/extensionGlobals'
 import { openUrl } from '../../../shared/utilities/vsCodeUtils'
 import { DefaultAmazonQAppInitContext } from '../../apps/initContext'
 
+const qChatModuleName = 'amazonqChat'
+
 export function dispatchWebViewMessagesToApps(
     webview: Webview,
     webViewToAppsMessagePublishers: Map<TabType, MessagePublisher<any>>
@@ -29,8 +31,8 @@ export function dispatchWebViewMessagesToApps(
                  * This would be equivalent of the duration between "user clicked open q" and "ui has become available"
                  * NOTE: Amazon Q UI is only loaded ONCE. The state is saved between each hide/show of the webview.
                  */
-                telemetry.webview_load.emit({
-                    webviewName: 'amazonq',
+                telemetry.toolkit_didLoadModule.emit({
+                    module: qChatModuleName,
                     duration: performance.measure(amazonqMark.uiReady, amazonqMark.open).duration,
                     result: 'Succeeded',
                 })
@@ -86,12 +88,19 @@ export function dispatchWebViewMessagesToApps(
         }
 
         if (msg.type === 'error') {
-            const event = msg.event === 'webview_load' ? telemetry.webview_load : telemetry.webview_error
-            event.emit({
-                webviewName: 'amazonqChat',
-                result: 'Failed',
-                reasonDesc: msg.errorMessage,
-            })
+            if (msg.event === 'toolkit_didLoadModule') {
+                telemetry.toolkit_didLoadModule.emit({
+                    module: qChatModuleName,
+                    result: 'Failed',
+                    reasonDesc: msg.errorMessage,
+                })
+            } else {
+                telemetry.webview_error.emit({
+                    webviewName: qChatModuleName,
+                    result: 'Failed',
+                    reasonDesc: msg.errorMessage,
+                })
+            }
             return
         }
 

packages/core/src/amazonq/webview/ui/main.ts

@@ -63,7 +63,7 @@ export const createMynahUI = (
         const { error, message } = e
         ideApi.postMessage({
             type: 'error',
-            event: connector.isUIReady ? 'webview_error' : 'webview_load',
+            event: connector.isUIReady ? 'webview_error' : 'toolkit_didLoadModule',
             errorMessage: error ? error.toString() : message,
         })
     })

packages/core/src/login/webview/vue/amazonq/backend_amazonq.ts

@@ -27,6 +27,7 @@ const className = 'AmazonQLoginWebview'
 export class AmazonQLoginWebview extends CommonAuthWebview {
     public override id: string = 'aws.amazonq.AmazonCommonAuth'
     public static sourcePath: string = 'vue/src/login/webview/vue/amazonq/index.js'
+    public override supportsLoadTelemetry: boolean = true
 
     override onActiveConnectionModified = new vscode.EventEmitter<void>()
 

packages/core/src/login/webview/vue/backend.ts

@@ -62,18 +62,25 @@ export abstract class CommonAuthWebview extends VueWebview {
     }
 
     private didCall: { login: boolean; reauth: boolean } = { login: false, reauth: false }
-    public setUiReady(state: 'login' | 'reauth') {
-        // Prevent telemetry spam, since showing/hiding chat triggers this each time.
-        // So only emit once.
+    /**
+     * Called when the UI load process is completed, regardless of success or failure
+     *
+     * @param errorMessage IF an error is caught on the frontend, this is the message. It will result in a failure metric.
+     *                     Otherwise we assume success.
+     */
+    public setUiReady(state: 'login' | 'reauth', errorMessage?: string) {
+        // Only emit once to prevent telemetry spam, since showing/hiding chat triggers this each time.
+        // TODO: Research how to not trigger this on every show/hide
         if (this.didCall[state]) {
             return
         }
 
-        telemetry.webview_load.emit({
-            passive: true,
-            webviewName: state,
-            result: 'Succeeded',
-        })
+        if (errorMessage) {
+            this.setLoadFailure(state, errorMessage)
+        } else {
+            this.setDidLoad(state)
+        }
+
         this.didCall[state] = true
     }
 

packages/core/src/login/webview/vue/login.vue

@@ -145,6 +145,7 @@
                 ></SelectableItem>
                 <button
                     class="continue-button"
+                    id="connection-selection-continue-button"
                     :disabled="selectedLoginOption === 0"
                     v-on:click="handleContinueClick()"
                 >
@@ -368,8 +369,6 @@ export default defineComponent({
         // Pre-select the first available login option
         await this.preselectLoginOption()
         await this.handleUrlInput() // validate the default startUrl
-
-        await client.setUiReady('login')
     },
     methods: {
         toggleItemSelection(itemId: number) {
@@ -590,6 +589,18 @@ export default defineComponent({
         },
     },
 })
+
+/**
+ * The ID of the element we will use to determine that the UI has completed its initial load.
+ *
+ * This makes assumptions that we will be in a certain state of the UI (eg showing a form vs. a loading bar).
+ * So if the UI flow changes, this may need to be updated.
+ */
+export function getReadyElementId() {
+    // On every initial load, we ASSUME that the user will always be in the connection selection state,
+    // which is why we specifically look for this button.
+    return 'connection-selection-continue-button'
+}
 </script>
 
 <style>

packages/core/src/login/webview/vue/reauthenticate.vue

@@ -61,7 +61,7 @@ and the final results are retrieved by the frontend. For this Component to updat
                 </div>
 
                 <div>
-                    <button id="reauthenticate" v-on:click="reauthenticate">Re-authenticate</button>
+                    <button id="reauthenticate-button" v-on:click="reauthenticate">Re-authenticate</button>
                     <div v-if="errorMessage" id="error-message" style="color: red">{{ errorMessage }}</div>
                 </div>
 
@@ -127,9 +127,6 @@ export default defineComponent({
 
         this.doShow = true
     },
-    async mounted() {
-        await client.setUiReady('reauth')
-    },
     methods: {
         async reauthenticate() {
             client.emitUiClick('auth_reauthenticate')
@@ -148,6 +145,16 @@ export default defineComponent({
         },
     },
 })
+
+/**
+ * The ID of the element we will use to determine that the UI has completed its initial load.
+ *
+ * This makes assumptions that we will be in a certain state of the UI (eg showing a form vs. a loading bar).
+ * So if the UI flow changes, this may need to be updated.
+ */
+export function getReadyElementId() {
+    return 'reauthenticate-button'
+}
 </script>
 <style>
 @import './base.css';
@@ -199,7 +206,7 @@ export default defineComponent({
     flex-direction: column;
 }
 
-button#reauthenticate {
+button#reauthenticate-button {
     cursor: pointer;
     background-color: var(--vscode-button-background);
     color: white;
@@ -231,7 +238,7 @@ button#cancel {
     cursor: pointer;
 }
 
-body.vscode-high-contrast:not(body.vscode-high-contrast-light) button#reauthenticate {
+body.vscode-high-contrast:not(body.vscode-high-contrast-light) button#reauthenticate-button {
     background-color: white;
     color: black;
 }

packages/core/src/login/webview/vue/root.vue

@@ -16,8 +16,8 @@ configure app to AMAZONQ if for Amazon Q login
 </template>
 <script lang="ts">
 import { PropType, defineComponent } from 'vue'
-import Login from './login.vue'
-import Reauthenticate from './reauthenticate.vue'
+import Login, { getReadyElementId as getLoginReadyElementId } from './login.vue'
+import Reauthenticate, { getReadyElementId as getReauthReadyElementId } from './reauthenticate.vue'
 import { AuthFlowState, FeatureId } from './types'
 import { WebviewClientFactory } from '../../../webviews/client'
 import { CommonAuthWebview } from './backend'
@@ -51,15 +51,83 @@ export default defineComponent({
         })
 
         await this.refreshAuthState()
+
+        // We were recieving the 'load' event before refreshAuthState() resolved (I'm assuming behavior w/ Vue + browser loading not blocking),
+        // so post refreshAuhState() if we detect we already loaded, then execute immediately since the event already happened.
+        if (didLoad) {
+            handleLoaded()
+        } else {
+            window.addEventListener('load', () => {
+                handleLoaded()
+            })
+        }
     },
     methods: {
         async refreshAuthState() {
             await client.refreshAuthState()
             this.authFlowState = await client.getAuthState()
+
+            // Used for telemetry purposes
+            if (this.authFlowState === 'LOGIN') {
+                ;(window as any).uiState = 'login'
+                ;(window as any).uiReadyElementId = getLoginReadyElementId()
+            } else if (this.authFlowState && this.authFlowState !== undefined) {
+                ;(window as any).uiState = 'reauth'
+                ;(window as any).uiReadyElementId = getReauthReadyElementId()
+            }
+
             this.refreshKey += 1
         },
     },
 })
+
+// ---- START ---- The following handles the process of indicating the UI has loaded successfully.
+// TODO: Move this in to a reusable class for other webviews, it feels a bit messy here
+let didSetReady = false
+
+// Setup error handlers to report 
+window.onerror = function (message) {
+    if (didSetReady) {
+        return
+    }
+
+    setUiReady((window as any).uiState, message.toString())
+}
+document.addEventListener(
+    'error',
+    (e) => {
+        if (didSetReady) {
+            return
+        }
+
+        setUiReady((window as any).uiState, e.message)
+    },
+    true
+)
+
+let didLoad = false
+window.addEventListener('load', () => {
+    didLoad = true
+})
+const handleLoaded = () => {
+    // in case some unexpected behavior triggers this flow again, skip since we already emitted for this instance
+    if (didSetReady) {
+        return
+    }
+
+    const foundElement = !!document.getElementById((window as any).uiReadyElementId)
+    if (!foundElement) {
+        setUiReady((window as any).uiState, `Could not find element: ${(window as any).uiReadyElementId}`)
+    } else {
+        // Successful load!    
+        setUiReady((window as any).uiState)
+    }
+}
+const setUiReady = (state: 'login' | 'reauth', errorMessage?: string) => {
+    client.setUiReady(state, errorMessage)
+    didSetReady = true
+}
+// ---- END ----
 </script>
 <style>
 body {

packages/core/src/shared/telemetry/vscodeTelemetry.json

@@ -985,16 +985,6 @@
             ],
             "passive": true
         },
-        {
-            "name": "webview_load",
-            "description": "Represents a webview load event",
-            "metadata": [
-                {
-                    "type": "webviewName",
-                    "required": true
-                }
-            ]
-        },
         {
             "name": "webview_error",
             "description": "Represents an error that occurs in a webview",