feat(telemetry): emit on opt-out and opt-in (aws#5259)

- Emits 1 final metric on opt-out that will let us know the user opted out.
- Also, on opt out it will emit any telemetry that was previously recorded and not sent.

Author: hayemaxi

packages/amazonq/test/unit/codewhisperer/service/codewhisperer.test.ts

@@ -36,9 +36,9 @@ describe('codewhisperer', async function () {
         telemetryEnabledDefault = globals.telemetry.telemetryEnabled
     })
 
-    afterEach(function () {
+    afterEach(async function () {
         sinon.restore()
-        globals.telemetry.telemetryEnabled = telemetryEnabledDefault
+        await globals.telemetry.setTelemetryEnabled(telemetryEnabledDefault)
     })
 
     it('sendTelemetryEvent for userTriggerDecision should respect telemetry optout status', async function () {
@@ -137,7 +137,7 @@ describe('codewhisperer', async function () {
         } as Request<SendTelemetryEventResponse, AWSError>)
 
         const authUtilStub = sinon.stub(AuthUtil.instance, 'isValidEnterpriseSsoInUse').returns(isSso)
-        globals.telemetry.telemetryEnabled = isTelemetryEnabled
+        await globals.telemetry.setTelemetryEnabled(isTelemetryEnabled)
         await codeWhispererClient.sendTelemetryEvent({ telemetryEvent: payload })
         const expectedOptOutPreference = isTelemetryEnabled ? 'OPTIN' : 'OPTOUT'
         if (isSso || isTelemetryEnabled) {

packages/amazonq/test/unit/codewhisperer/tracker/codewhispererTracker.test.ts

@@ -34,13 +34,13 @@ describe('codewhispererTracker', function () {
             assert.ok(!pushSpy.neverCalledWith(suggestion))
         })
 
-        it('Should not enque when telemetry is disabled', function () {
-            globals.telemetry.telemetryEnabled = false
+        it('Should not enque when telemetry is disabled', async function () {
+            await globals.telemetry.setTelemetryEnabled(false)
             const suggestion = createAcceptedSuggestionEntry()
             const pushSpy = sinon.spy(Array.prototype, 'push')
             CodeWhispererTracker.getTracker().enqueue(suggestion)
             assert.ok(pushSpy.neverCalledWith(suggestion))
-            globals.telemetry.telemetryEnabled = true
+            await globals.telemetry.setTelemetryEnabled(true)
         })
     })
 
@@ -67,11 +67,11 @@ describe('codewhispererTracker', function () {
         })
 
         it('Should skip if telemetry is disabled', async function () {
-            globals.telemetry.telemetryEnabled = false
+            await globals.telemetry.setTelemetryEnabled(false)
             const getTimeSpy = sinon.spy(Date.prototype, 'getTime')
             await CodeWhispererTracker.getTracker().flush()
             assert.ok(!getTimeSpy.called)
-            globals.telemetry.telemetryEnabled = true
+            await globals.telemetry.setTelemetryEnabled(true)
         })
     })
 

packages/amazonq/test/unit/codewhisperer/util/editorContext.test.ts

@@ -18,8 +18,8 @@ describe('editorContext', function () {
         telemetryEnabledDefault = globals.telemetry.telemetryEnabled
     })
 
-    afterEach(function () {
-        globals.telemetry.telemetryEnabled = telemetryEnabledDefault
+    afterEach(async function () {
+        await globals.telemetry.setTelemetryEnabled(telemetryEnabledDefault)
     })
 
     describe('extractContextForCodeWhisperer', function () {
@@ -136,7 +136,7 @@ describe('editorContext', function () {
         it('Should return expected fields for optOut, nextToken and reference config', async function () {
             const nextToken = 'testToken'
             const optOutPreference = false
-            globals.telemetry.telemetryEnabled = false
+            await globals.telemetry.setTelemetryEnabled(false)
             const editor = createMockTextEditor('import math\ndef two_sum(nums, target):\n', 'test.py', 'python', 1, 17)
             const actual = await EditorContext.buildListRecommendationRequest(editor, nextToken, optOutPreference)
 

packages/core/src/shared/telemetry/activation.ts

@@ -18,6 +18,7 @@ import { TelemetryConfig, setupTelemetryId } from './util'
 import { isAutomation, isReleaseVersion } from '../vscode/env'
 import { AWSProduct } from './clienttelemetry'
 import { DefaultTelemetryClient } from './telemetryClient'
+import { telemetry } from './telemetry'
 import { Commands } from '../vscode/commands2'
 
 export const noticeResponseViewSettings = localize('AWS.telemetry.notificationViewSettings', 'Settings')
@@ -48,18 +49,32 @@ export async function activate(
     DefaultTelemetryClient.productName = productName
     globals.telemetry = await DefaultTelemetryService.create(extensionContext, awsContext, getComputeRegion())
 
+    const isAmazonQExt = isAmazonQ()
     try {
-        globals.telemetry.telemetryEnabled = config.isEnabled()
+        await globals.telemetry.setTelemetryEnabled(config.isEnabled())
 
         extensionContext.subscriptions.push(
-            (isAmazonQ() ? config.amazonQConfig : config.toolkitConfig).onDidChange(event => {
+            (isAmazonQExt ? config.amazonQConfig : config.toolkitConfig).onDidChange(async event => {
                 if (event.key === 'telemetry') {
-                    globals.telemetry.telemetryEnabled = config.isEnabled()
+                    const val = config.isEnabled()
+                    const settingId = isAmazonQExt ? 'amazonQ.telemetry' : 'aws.telemetry'
+
+                    // Record 'disabled' right before its turned off, so we can send this + the batch we have already.
+                    if (!val) {
+                        telemetry.aws_modifySetting.emit({ settingId, settingState: 'false', result: 'Succeeded' })
+                    }
+
+                    await globals.telemetry.setTelemetryEnabled(val)
+
+                    // Record 'enabled' after its turned on, otherwise this is ignored.
+                    if (val) {
+                        telemetry.aws_modifySetting.emit({ settingId, settingState: 'true', result: 'Succeeded' })
+                    }
                 }
             })
         )
 
-        if (isAmazonQ()) {
+        if (isAmazonQExt) {
             extensionContext.subscriptions.push(
                 Commands.register('aws.amazonq.setupTelemetryId', async () => {
                     await setupTelemetryId(extensionContext)

packages/core/src/shared/telemetry/telemetryService.ts

@@ -112,16 +112,16 @@ export class DefaultTelemetryService {
     public get telemetryEnabled(): boolean {
         return this._telemetryEnabled
     }
-    public set telemetryEnabled(value: boolean) {
+    public async setTelemetryEnabled(value: boolean) {
         if (this._telemetryEnabled !== value) {
-            getLogger().verbose(`Telemetry is now ${value ? 'enabled' : 'disabled'}`)
-        }
+            this._telemetryEnabled = value
 
-        // clear the queue on explicit disable
-        if (!value) {
-            this.clearRecords()
+            // send all the gathered data that the user was opted-in for, prior to disabling
+            if (!value) {
+                await this._flushRecords()
+            }
         }
-        this._telemetryEnabled = value
+        getLogger().verbose(`Telemetry is ${value ? 'enabled' : 'disabled'}`)
     }
 
     public get timer(): NodeJS.Timer | undefined {
@@ -172,16 +172,26 @@ export class DefaultTelemetryService {
         }
     }
 
+    /**
+     * Publish metrics to the Telemetry Service.
+     */
     private async flushRecords(): Promise<void> {
         if (this.telemetryEnabled) {
-            if (this.publisher === undefined) {
-                await this.createDefaultPublisherAndClient()
-            }
-            if (this.publisher !== undefined) {
-                this.publisher.enqueue(...this._eventQueue)
-                await this.publisher.flush()
-                this.clearRecords()
-            }
+            await this._flushRecords()
+        }
+    }
+
+    /**
+     * @warning DO NOT USE DIRECTLY, use `flushRecords()` instead.
+     */
+    private async _flushRecords(): Promise<void> {
+        if (this.publisher === undefined) {
+            await this.createDefaultPublisherAndClient()
+        }
+        if (this.publisher !== undefined) {
+            this.publisher.enqueue(...this._eventQueue)
+            await this.publisher.flush()
+            this.clearRecords()
         }
     }
 

packages/core/src/test/globalSetup.test.ts

@@ -79,7 +79,7 @@ export const mochaHooks = {
         }
 
         // Enable telemetry features for tests. The metrics won't actually be posted.
-        globals.telemetry.telemetryEnabled = true
+        await globals.telemetry.setTelemetryEnabled(true)
         globals.telemetry.clearRecords()
         globals.telemetry.logger.clear()
         TelemetryDebounceInfo.instance.clear()

packages/core/src/test/shared/telemetry/telemetryService.test.ts

@@ -47,7 +47,7 @@ describe('TelemetryService', function () {
     async function initService(awsContext = new FakeAwsContext()): Promise<DefaultTelemetryService> {
         const newService = await DefaultTelemetryService.create(mockContext, awsContext, undefined, mockPublisher)
         newService.flushPeriod = testFlushPeriod
-        newService.telemetryEnabled = true
+        await newService.setTelemetryEnabled(true)
 
         return newService
     }
@@ -80,7 +80,7 @@ describe('TelemetryService', function () {
     })
 
     it('posts feedback', async function () {
-        service.telemetryEnabled = false
+        await service.setTelemetryEnabled(false)
         const feedback = { comment: '', sentiment: '' }
         await service.postFeedback(feedback)
 
@@ -238,23 +238,24 @@ describe('TelemetryService', function () {
         assertMetadataContainsTestAccount(metricData[0], AccountStatus.NotSet)
     })
 
-    it('events are never recorded if telemetry has been disabled', async function () {
+    it('new events are not recorded if telemetry has been disabled', async function () {
         stubGlobal()
 
-        service.telemetryEnabled = false
+        service.record(fakeMetric({ metricName: 'first' })) // emits 1 metric
+        await service.setTelemetryEnabled(false)
         await service.start()
 
-        // telemetry off: events are never recorded
-        service.record(fakeMetric({ metricName: 'name' }))
+        // telemetry off: new events are never recorded
+        service.record(fakeMetric({ metricName: 'second' })) // would emit 1 metric
         await service.shutdown()
         await clock.tickAsync(testFlushPeriod * 2)
 
-        // events are never flushed
-        assert.strictEqual(mockPublisher.flushCount, 0)
-        assert.strictEqual(mockPublisher.enqueueCount, 0)
-        assert.strictEqual(mockPublisher.queue.length, 0)
-        // and events are not kept in memory
-        assert.strictEqual(logger.metricCount, 0)
+        // only the events prior to disabling are recorded
+        assert.strictEqual(mockPublisher.flushCount, 1)
+        assert.strictEqual(mockPublisher.enqueueCount, 1)
+        assert.strictEqual(mockPublisher.queue.length, 1)
+        // and newer events are not kept in memory
+        assert.strictEqual(logger.metricCount, 1)
     })
 
     function assertMetadataContainsTestAccount(

packages/core/src/testInteg/schema/schema.test.ts

@@ -73,7 +73,7 @@ describe('getDefaultSchemas()', () => {
 
     it('uses cache after initial fetch for CFN/SAM schema', async () => {
         fs.removeSync(GlobalStorage.samAndCfnSchemaDestinationUri().fsPath)
-        globals.telemetry.telemetryEnabled = true
+        await globals.telemetry.setTelemetryEnabled(true)
         globals.telemetry.clearRecords()
         globals.telemetry.logger.clear()
         await getDefaultSchemas()