telemetry(auth): Update metrics to better debug Auth dropoff

packages/amazonq/src/extension.ts

@@ -142,7 +142,7 @@ export async function activateAmazonQCommon(context: vscode.ExtensionContext, is
         // Give time for the extension to finish initializing.
         globals.clock.setTimeout(async () => {
             CommonAuthWebview.authSource = ExtStartUpSources.firstStartUp
-            void focusAmazonQPanel.execute(placeholder, 'firstStartUp')
+            void focusAmazonQPanel.execute(placeholder, ExtStartUpSources.firstStartUp)
         }, 1000)
     }
 }

packages/core/src/auth/utils.ts

@@ -58,7 +58,7 @@ import { EcsCredentialsProvider } from './providers/ecsCredentialsProvider'
 import { EnvVarsCredentialsProvider } from './providers/envVarsCredentialsProvider'
 import { showMessageWithUrl } from '../shared/utilities/messages'
 import { credentialHelpUrl } from '../shared/constants'
-import { ExtStartUpSources, ExtStartUpSource } from '../shared/telemetry/util'
+import { ExtStartUpSources, ExtStartUpSource, hadClientIdOnStartup } from '../shared/telemetry/util'
 
 // iam-only excludes Builder ID and IAM Identity Center from the list of valid connections
 // TODO: Understand if "iam" should include these from the list at all
@@ -688,17 +688,43 @@ export class ExtensionUse {
             return this.isFirstUseCurrentSession
         }
 
-        this.isFirstUseCurrentSession = globals.globalState.get('isExtensionFirstUse')
-        if (this.isFirstUseCurrentSession === undefined) {
+        // This is for sure not their first use
+        const isFirstUse = globals.globalState.tryGet('isExtensionFirstUse', Boolean)
+        if (isFirstUse === false) {
+            this.isFirstUseCurrentSession = isFirstUse
+            return this.isFirstUseCurrentSession
+        }
+
+        /**
+         * SANITY CHECK: If the clientId already existed on startup, then isFirstUse MUST be false. So
+         * there is a bug in the state.
+         */
+        if (hadClientIdOnStartup(globals.globalState)) {
+            telemetry.function_call.emit({
+                result: 'Failed',
+                functionName: 'isFirstUse',
+                reason: 'ClientIdAlreadyExisted',
+            })
+        }
+
+        if (isAmazonQ()) {
+            this.isFirstUseCurrentSession = true
+            if (hasExistingConnections()) {
+                telemetry.function_call.emit({
+                    result: 'Failed',
+                    functionName: 'isFirstUse',
+                    reason: 'UnexpectedConnections',
+                })
+            }
+        } else {
             // The variable in the store is not defined yet, fallback to checking if they have existing connections.
             this.isFirstUseCurrentSession = !hasExistingConnections()
-
-            getLogger().debug(
-                `isFirstUse: State not found, marking user as '${
-                    this.isFirstUseCurrentSession ? '' : 'NOT '
-                }first use' since they 'did ${this.isFirstUseCurrentSession ? 'NOT ' : ''}have existing connections'.`
-            )
         }
+        getLogger().debug(
+            `isFirstUse: State not found, marking user as '${
+                this.isFirstUseCurrentSession ? '' : 'NOT '
+            }first use' since they 'did ${this.isFirstUseCurrentSession ? 'NOT ' : ''}have existing connections'.`
+        )
 
         // Update state, so next time it is not first use
         this.updateMemento('isExtensionFirstUse', false)

packages/core/src/login/webview/commonAuthViewProvider.ts

@@ -46,6 +46,7 @@ import { AuthSources } from './util'
 import { AuthFlowStates } from './vue/types'
 import { getTelemetryMetadataForConn } from '../../auth/connection'
 import { AuthUtil } from '../../codewhisperer/util/authUtil'
+import { ExtensionUse } from '../../auth/utils'
 
 export class CommonAuthViewProvider implements WebviewViewProvider {
     public readonly viewType: string
@@ -83,14 +84,22 @@ export class CommonAuthViewProvider implements WebviewViewProvider {
     ) {
         // Our callback won't fire on the first view.
         if (webviewView.visible) {
-            telemetry.auth_signInPageOpened.emit({ result: 'Succeeded', passive: true })
+            telemetry.auth_signInPageOpened.emit({
+                result: 'Succeeded',
+                passive: true,
+                source: ExtensionUse.instance.sourceForTelemetry(),
+            })
         }
 
         // This will fire whenever the user opens or closes the login page from 'somewhere else'
         // i.e. NOT when switching from/to the chat window, which uses the same view area.
         webviewView.onDidChangeVisibility(async () => {
             if (webviewView.visible) {
-                telemetry.auth_signInPageOpened.emit({ result: 'Succeeded', passive: true })
+                telemetry.auth_signInPageOpened.emit({
+                    result: 'Succeeded',
+                    passive: true,
+                    source: ExtensionUse.instance.sourceForTelemetry(),
+                })
             } else {
                 telemetry.auth_signInPageClosed.emit({ result: 'Succeeded', passive: true })
 

packages/core/src/login/webview/vue/backend.ts

@@ -61,6 +61,22 @@ export abstract class CommonAuthWebview extends VueWebview {
         return globals.regionProvider.getRegions().reverse()
     }
 
+    private didCall: { login: boolean; reauth: boolean } = { login: false, reauth: false }
+    public setUiReady(state: 'login' | 'reauth') {
+        // Prevent telemetry spam, since showing/hiding chat triggers this each time.
+        // So only emit once.
+        if (this.didCall[state]) {
+            return
+        }
+
+        telemetry.webview_load.emit({
+            passive: true,
+            webviewName: state,
+            result: 'Succeeded',
+        })
+        this.didCall[state] = true
+    }
+
     /**
      * This wraps the execution of the given setupFunc() and handles common errors from the SSO setup process.
      *

packages/core/src/login/webview/vue/login.vue

@@ -368,6 +368,8 @@ export default defineComponent({
         // Pre-select the first available login option
         await this.preselectLoginOption()
         await this.handleUrlInput() // validate the default startUrl
+
+        await client.setUiReady('login')
     },
     methods: {
         toggleItemSelection(itemId: number) {

packages/core/src/login/webview/vue/reauthenticate.vue

@@ -127,6 +127,9 @@ export default defineComponent({
 
         this.doShow = true
     },
+    async mounted() {
+        await client.setUiReady('reauth')
+    },
     methods: {
         async reauthenticate() {
             client.emitUiClick('auth_reauthenticate')

packages/core/src/shared/telemetry/telemetryService.ts

@@ -22,6 +22,7 @@ import { telemetry, MetricBase } from './telemetry'
 import fs from '../fs/fs'
 import fsNode from 'fs/promises'
 import * as collectionUtil from '../utilities/collectionUtils'
+import { ExtensionUse } from '../../auth/utils'
 
 export type TelemetryService = ClassToInterfaceType<DefaultTelemetryService>
 
@@ -98,7 +99,9 @@ export class DefaultTelemetryService {
         // TODO: `readEventsFromCache` should be async
         this._eventQueue.push(...(await DefaultTelemetryService.readEventsFromCache(this.persistFilePath)))
         this._endOfCache = this._eventQueue[this._eventQueue.length - 1]
-        telemetry.session_start.emit()
+        telemetry.session_start.emit({
+            source: ExtensionUse.instance.sourceForTelemetry(),
+        })
         this.startFlushInterval()
     }
 

packages/core/src/shared/telemetry/util.ts

@@ -30,6 +30,7 @@ import { asStringifiedStack, FunctionEntry } from './spans'
 import { telemetry } from './telemetry'
 import { v5 as uuidV5 } from 'uuid'
 import { ToolkitError } from '../errors'
+import { GlobalState } from '../globalState'
 
 const legacySettingsTelemetryValueDisable = 'Disable'
 const legacySettingsTelemetryValueEnable = 'Enable'
@@ -177,6 +178,8 @@ export const getClientId = memoize(
             const localClientId = globalState.tryGet('telemetryClientId', String) // local to extension, despite accessing "global" state
             let clientId: string
 
+            _hadClientIdOnStartup = !!globalClientId || !!localClientId
+
             if (isWeb()) {
                 const machineId = vscode.env.machineId
                 clientId = localClientId ?? machineId
@@ -210,6 +213,22 @@ export const getClientId = memoize(
     }
 )
 
+let _hadClientIdOnStartup = false
+/**
+ * Returns true if the ClientID existed before this session started
+ */
+export const hadClientIdOnStartup = (
+    globalState: GlobalState,
+    update = (globalState: GlobalState) => {
+        getClientId(globalState)
+    }
+) => {
+    // triggers the flow that will update the state, if not done already
+    update(globalState)
+
+    return _hadClientIdOnStartup
+}
+
 export const platformPair = () => `${env.appName.replace(/\s/g, '-')}/${version}`
 
 /**

packages/core/src/shared/telemetry/vscodeTelemetry.json

@@ -1131,6 +1131,17 @@
             ],
             "passive": true
         },
+        {
+            "name": "auth_signInPageOpened",
+            "description": "When the Amazon Q sign in page is opened and focused.",
+            "metadata": [
+                {
+                    "type": "source",
+                    "required": true
+                }
+            ],
+            "passive": true
+        },
         {
             "name": "function_call",
             "description": "Represents a function call. In most cases this should wrap code with a run(), then you can add context.",
@@ -1207,6 +1218,17 @@
                 }
             ]
         },
+        {
+            "name": "session_start",
+            "description": "Called when starting the plugin",
+            "metadata": [
+                {
+                    "type": "source",
+                    "required": false
+                }
+            ],
+            "passive": true
+        },
         {
             "name": "session_end",
             "description": "Called when stopping the IDE on a best effort basis",

packages/core/src/test/credentials/utils.test.ts

@@ -11,23 +11,23 @@ import globals from '../../shared/extensionGlobals'
 
 describe('ExtensionUse.isFirstUse()', function () {
     let instance: ExtensionUse
+    const notHasExistingConnections = () => false
 
     beforeEach(async function () {
         instance = new ExtensionUse()
-        await globals.globalState.update(ExtensionUse.instance.isExtensionFirstUseKey, true)
+        await makeStateValueNotExist()
     })
 
     it('is true only on first startup', function () {
-        assert.strictEqual(instance.isFirstUse(), true, 'Failed on first call.')
-        assert.strictEqual(instance.isFirstUse(), true, 'Failed on second call.')
+        assert.strictEqual(instance.isFirstUse(notHasExistingConnections), true, 'Failed on first call.')
+        assert.strictEqual(instance.isFirstUse(notHasExistingConnections), true, 'Failed on second call.')
 
         const nextStartup = nextExtensionStartup()
-        assert.strictEqual(nextStartup.isFirstUse(), false, 'Failed on new startup.')
+        assert.strictEqual(nextStartup.isFirstUse(notHasExistingConnections), false, 'Failed on new startup.')
     })
 
     it('true when: (state value not exists + NOT has existing connections)', async function () {
         await makeStateValueNotExist()
-        const notHasExistingConnections = () => false
         assert.strictEqual(
             instance.isFirstUse(notHasExistingConnections),
             true,

packages/core/src/test/shared/telemetry/util.test.ts

@@ -10,6 +10,7 @@ import {
     convertLegacy,
     getClientId,
     getUserAgent,
+    hadClientIdOnStartup,
     platformPair,
     SessionId,
     telemetryClientIdEnvKey,
@@ -128,20 +129,29 @@ describe('getSessionId', function () {
 
 describe('getClientId', function () {
     before(function () {
-        delete process.env[telemetryClientIdEnvKey]
+        setClientIdEnvVar(undefined)
     })
 
     afterEach(function () {
-        delete process.env[telemetryClientIdEnvKey]
+        setClientIdEnvVar(undefined)
     })
 
     function testGetClientId(globalState: GlobalState) {
         return getClientId(globalState, true, false, randomUUID())
     }
 
+    function setClientIdEnvVar(val: string | undefined) {
+        if (val === undefined) {
+            delete process.env[telemetryClientIdEnvKey]
+            return
+        }
+
+        process.env[telemetryClientIdEnvKey] = val
+    }
+
     it('generates a unique id if no other id is available', function () {
         const c1 = testGetClientId(new GlobalState(new FakeMemento()))
-        delete process.env[telemetryClientIdEnvKey]
+        setClientIdEnvVar(undefined)
         const c2 = testGetClientId(new GlobalState(new FakeMemento()))
         assert.notStrictEqual(c1, c2)
     })
@@ -160,7 +170,7 @@ describe('getClientId', function () {
 
         const e = new GlobalState(new FakeMemento())
         await e.update('telemetryClientId', randomUUID())
-        process.env[telemetryClientIdEnvKey] = expectedClientId
+        setClientIdEnvVar(expectedClientId)
 
         assert.strictEqual(testGetClientId(new GlobalState(new FakeMemento())), expectedClientId)
     })
@@ -218,6 +228,25 @@ describe('getClientId', function () {
         const clientId = getClientId(new GlobalState(new FakeMemento()), false, false)
         assert.strictEqual(clientId, '11111111-1111-1111-1111-111111111111')
     })
+
+    describe('hadClientIdOnStartup', async function () {
+        it('returns false when no existing clientId', async function () {
+            const globalState = new GlobalState(new FakeMemento())
+            assert.strictEqual(hadClientIdOnStartup(globalState, testGetClientId), false)
+        })
+
+        it('returns true when existing env var clientId', async function () {
+            const globalState = new GlobalState(new FakeMemento())
+            setClientIdEnvVar('aaa-111')
+            assert.strictEqual(hadClientIdOnStartup(globalState, testGetClientId), true)
+        })
+
+        it('returns true when existing state clientId', async function () {
+            const globalState = new GlobalState(new FakeMemento())
+            await globalState.update('telemetryClientId', 'bbb-222')
+            assert.strictEqual(hadClientIdOnStartup(globalState, testGetClientId), true)
+        })
+    })
 })
 
 describe('getUserAgent', function () {