telemetry(auth): Update metrics to better debug Auth dropoff (#6625)

## Problem

We noticed that there was an auth dropoff between `session_start` with a
brand new clientId, versus when the `auth_userState` metric indicated
`isFirstUse` meaning the user is net new. We went from 12.7k for
`session_start` to 9.8k for `auth_userState`, these should have been
basically the same.

## Solution

Add in certain metrics to help debug where the discrepancy is coming
from:

- When we determine the user is a first time user, we will also check if
the clientId is newly generated. If this is not the case we know there
is a discrepancy here
- The relevant metric will be `function_call` with a `functionName:
isFirstUse`, `result: Failed`, and a `reason: ClientIdAlreadyExisted`
- When we determine the user is a first time user, if we detected that
they had previous auth connections, this will indicate a likely cause
for the discrepancy
- The relevant metric will be `function_call` with `reason:
UnexpectedConnections`
- We will emit metrics when the Auth Login page loads since that also
had a discrepancy and the telemetry did not exist
- The relevant metric is `webview_load` and it will indicate when the
Auth Login/Reauth page has actually loaded
- Previously we were observing the telemetry for the command
`aws.amazonq.focusChat`, but all this did was emit when called and
didn't confirm the UI actually loaded.
- We will also add the `isFirstUse` metric source value in to some other
existing metrics

---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

---------

Signed-off-by: nkomonen-amazon <[email protected]>

Author: nkomonen-amazon

packages/amazonq/src/extension.ts

@@ -142,7 +142,7 @@ export async function activateAmazonQCommon(context: vscode.ExtensionContext, is
         // Give time for the extension to finish initializing.
         globals.clock.setTimeout(async () => {
             CommonAuthWebview.authSource = ExtStartUpSources.firstStartUp
-            void focusAmazonQPanel.execute(placeholder, 'firstStartUp')
+            void focusAmazonQPanel.execute(placeholder, ExtStartUpSources.firstStartUp)
         }, 1000)
     }
 }

packages/core/src/auth/utils.ts

@@ -58,7 +58,7 @@ import { EcsCredentialsProvider } from './providers/ecsCredentialsProvider'
 import { EnvVarsCredentialsProvider } from './providers/envVarsCredentialsProvider'
 import { showMessageWithUrl } from '../shared/utilities/messages'
 import { credentialHelpUrl } from '../shared/constants'
-import { ExtStartUpSources, ExtStartUpSource } from '../shared/telemetry/util'
+import { ExtStartUpSources, ExtStartUpSource, hadClientIdOnStartup } from '../shared/telemetry/util'
 
 // iam-only excludes Builder ID and IAM Identity Center from the list of valid connections
 // TODO: Understand if "iam" should include these from the list at all
@@ -688,17 +688,43 @@ export class ExtensionUse {
             return this.isFirstUseCurrentSession
         }
 
-        this.isFirstUseCurrentSession = globals.globalState.get('isExtensionFirstUse')
-        if (this.isFirstUseCurrentSession === undefined) {
+        // This is for sure not their first use
+        const isFirstUse = globals.globalState.tryGet('isExtensionFirstUse', Boolean)
+        if (isFirstUse === false) {
+            this.isFirstUseCurrentSession = isFirstUse
+            return this.isFirstUseCurrentSession
+        }
+
+        /**
+         * SANITY CHECK: If the clientId already existed on startup, then isFirstUse MUST be false. So
+         * there is a bug in the state.
+         */
+        if (hadClientIdOnStartup(globals.globalState)) {
+            telemetry.function_call.emit({
+                result: 'Failed',
+                functionName: 'isFirstUse',
+                reason: 'ClientIdAlreadyExisted',
+            })
+        }
+
+        if (isAmazonQ()) {
+            this.isFirstUseCurrentSession = true
+            if (hasExistingConnections()) {
+                telemetry.function_call.emit({
+                    result: 'Failed',
+                    functionName: 'isFirstUse',
+                    reason: 'UnexpectedConnections',
+                })
+            }
+        } else {
             // The variable in the store is not defined yet, fallback to checking if they have existing connections.
             this.isFirstUseCurrentSession = !hasExistingConnections()
-
-            getLogger().debug(
-                `isFirstUse: State not found, marking user as '${
-                    this.isFirstUseCurrentSession ? '' : 'NOT '
-                }first use' since they 'did ${this.isFirstUseCurrentSession ? 'NOT ' : ''}have existing connections'.`
-            )
         }
+        getLogger().debug(
+            `isFirstUse: State not found, marking user as '${
+                this.isFirstUseCurrentSession ? '' : 'NOT '
+            }first use' since they 'did ${this.isFirstUseCurrentSession ? 'NOT ' : ''}have existing connections'.`
+        )
 
         // Update state, so next time it is not first use
         this.updateMemento('isExtensionFirstUse', false)

packages/core/src/login/webview/commonAuthViewProvider.ts

@@ -46,6 +46,7 @@ import { AuthSources } from './util'
 import { AuthFlowStates } from './vue/types'
 import { getTelemetryMetadataForConn } from '../../auth/connection'
 import { AuthUtil } from '../../codewhisperer/util/authUtil'
+import { ExtensionUse } from '../../auth/utils'
 
 export class CommonAuthViewProvider implements WebviewViewProvider {
     public readonly viewType: string
@@ -83,14 +84,22 @@ export class CommonAuthViewProvider implements WebviewViewProvider {
     ) {
         // Our callback won't fire on the first view.
         if (webviewView.visible) {
-            telemetry.auth_signInPageOpened.emit({ result: 'Succeeded', passive: true })
+            telemetry.auth_signInPageOpened.emit({
+                result: 'Succeeded',
+                passive: true,
+                source: ExtensionUse.instance.sourceForTelemetry(),
+            })
         }
 
         // This will fire whenever the user opens or closes the login page from 'somewhere else'
         // i.e. NOT when switching from/to the chat window, which uses the same view area.
         webviewView.onDidChangeVisibility(async () => {
             if (webviewView.visible) {
-                telemetry.auth_signInPageOpened.emit({ result: 'Succeeded', passive: true })
+                telemetry.auth_signInPageOpened.emit({
+                    result: 'Succeeded',
+                    passive: true,
+                    source: ExtensionUse.instance.sourceForTelemetry(),
+                })
             } else {
                 telemetry.auth_signInPageClosed.emit({ result: 'Succeeded', passive: true })
 

packages/core/src/login/webview/vue/backend.ts

@@ -61,6 +61,22 @@ export abstract class CommonAuthWebview extends VueWebview {
         return globals.regionProvider.getRegions().reverse()
     }
 
+    private didCall: { login: boolean; reauth: boolean } = { login: false, reauth: false }
+    public setUiReady(state: 'login' | 'reauth') {
+        // Prevent telemetry spam, since showing/hiding chat triggers this each time.
+        // So only emit once.
+        if (this.didCall[state]) {
+            return
+        }
+
+        telemetry.webview_load.emit({
+            passive: true,
+            webviewName: state,
+            result: 'Succeeded',
+        })
+        this.didCall[state] = true
+    }
+
     /**
      * This wraps the execution of the given setupFunc() and handles common errors from the SSO setup process.
      *

packages/core/src/login/webview/vue/login.vue

@@ -368,6 +368,8 @@ export default defineComponent({
         // Pre-select the first available login option
         await this.preselectLoginOption()
         await this.handleUrlInput() // validate the default startUrl
+
+        await client.setUiReady('login')
     },
     methods: {
         toggleItemSelection(itemId: number) {

packages/core/src/login/webview/vue/reauthenticate.vue

@@ -127,6 +127,9 @@ export default defineComponent({
 
         this.doShow = true
     },
+    async mounted() {
+        await client.setUiReady('reauth')
+    },
     methods: {
         async reauthenticate() {
             client.emitUiClick('auth_reauthenticate')

packages/core/src/shared/telemetry/telemetryService.ts

@@ -22,6 +22,7 @@ import { telemetry, MetricBase } from './telemetry'
 import fs from '../fs/fs'
 import fsNode from 'fs/promises'
 import * as collectionUtil from '../utilities/collectionUtils'
+import { ExtensionUse } from '../../auth/utils'
 
 export type TelemetryService = ClassToInterfaceType<DefaultTelemetryService>
 
@@ -98,7 +99,9 @@ export class DefaultTelemetryService {
         // TODO: `readEventsFromCache` should be async
         this._eventQueue.push(...(await DefaultTelemetryService.readEventsFromCache(this.persistFilePath)))
         this._endOfCache = this._eventQueue[this._eventQueue.length - 1]
-        telemetry.session_start.emit()
+        telemetry.session_start.emit({
+            source: ExtensionUse.instance.sourceForTelemetry(),
+        })
         this.startFlushInterval()
     }
 

packages/core/src/shared/telemetry/util.ts

@@ -30,6 +30,7 @@ import { asStringifiedStack, FunctionEntry } from './spans'
 import { telemetry } from './telemetry'
 import { v5 as uuidV5 } from 'uuid'
 import { ToolkitError } from '../errors'
+import { GlobalState } from '../globalState'
 
 const legacySettingsTelemetryValueDisable = 'Disable'
 const legacySettingsTelemetryValueEnable = 'Enable'
@@ -177,6 +178,8 @@ export const getClientId = memoize(
             const localClientId = globalState.tryGet('telemetryClientId', String) // local to extension, despite accessing "global" state
             let clientId: string
 
+            _hadClientIdOnStartup = !!globalClientId || !!localClientId
+
             if (isWeb()) {
                 const machineId = vscode.env.machineId
                 clientId = localClientId ?? machineId
@@ -210,6 +213,22 @@ export const getClientId = memoize(
     }
 )
 
+let _hadClientIdOnStartup = false
+/**
+ * Returns true if the ClientID existed before this session started
+ */
+export const hadClientIdOnStartup = (
+    globalState: GlobalState,
+    update = (globalState: GlobalState) => {
+        getClientId(globalState)
+    }
+) => {
+    // triggers the flow that will update the state, if not done already
+    update(globalState)
+
+    return _hadClientIdOnStartup
+}
+
 export const platformPair = () => `${env.appName.replace(/\s/g, '-')}/${version}`
 
 /**

packages/core/src/shared/telemetry/vscodeTelemetry.json

@@ -1131,6 +1131,17 @@
             ],
             "passive": true
         },
+        {
+            "name": "auth_signInPageOpened",
+            "description": "When the Amazon Q sign in page is opened and focused.",
+            "metadata": [
+                {
+                    "type": "source",
+                    "required": true
+                }
+            ],
+            "passive": true
+        },
         {
             "name": "function_call",
             "description": "Represents a function call. In most cases this should wrap code with a run(), then you can add context.",
@@ -1207,6 +1218,17 @@
                 }
             ]
         },
+        {
+            "name": "session_start",
+            "description": "Called when starting the plugin",
+            "metadata": [
+                {
+                    "type": "source",
+                    "required": false
+                }
+            ],
+            "passive": true
+        },
         {
             "name": "session_end",
             "description": "Called when stopping the IDE on a best effort basis",

packages/core/src/test/credentials/utils.test.ts

@@ -11,23 +11,23 @@ import globals from '../../shared/extensionGlobals'
 
 describe('ExtensionUse.isFirstUse()', function () {
     let instance: ExtensionUse
+    const notHasExistingConnections = () => false
 
     beforeEach(async function () {
         instance = new ExtensionUse()
-        await globals.globalState.update(ExtensionUse.instance.isExtensionFirstUseKey, true)
+        await makeStateValueNotExist()
     })
 
     it('is true only on first startup', function () {
-        assert.strictEqual(instance.isFirstUse(), true, 'Failed on first call.')
-        assert.strictEqual(instance.isFirstUse(), true, 'Failed on second call.')
+        assert.strictEqual(instance.isFirstUse(notHasExistingConnections), true, 'Failed on first call.')
+        assert.strictEqual(instance.isFirstUse(notHasExistingConnections), true, 'Failed on second call.')
 
         const nextStartup = nextExtensionStartup()
-        assert.strictEqual(nextStartup.isFirstUse(), false, 'Failed on new startup.')
+        assert.strictEqual(nextStartup.isFirstUse(notHasExistingConnections), false, 'Failed on new startup.')
     })
 
     it('true when: (state value not exists + NOT has existing connections)', async function () {
         await makeStateValueNotExist()
-        const notHasExistingConnections = () => false
         assert.strictEqual(
             instance.isFirstUse(notHasExistingConnections),
             true,