Add cloudformation usage metrics (#308)

Author: satyakigh

src/autocomplete/CompletionRouter.ts

@@ -1,4 +1,4 @@
-import { CompletionParams } from 'vscode-languageserver';
+import { CompletionItem, CompletionParams } from 'vscode-languageserver';
 import { Context } from '../context/Context';
 import { ContextManager } from '../context/ContextManager';
 import {
@@ -20,6 +20,7 @@ import { SettingsConfigurable, ISettingsSubscriber, SettingsSubscription } from
 import { CompletionSettings, DefaultSettings } from '../settings/Settings';
 import { LoggerFactory } from '../telemetry/LoggerFactory';
 import { Track } from '../telemetry/TelemetryDecorator';
+import { EventType, UsageTracker } from '../usageTracker/UsageTracker';
 import { Closeable } from '../utils/Closeable';
 import { CompletionFormatter } from './CompletionFormatter';
 import { CompletionProvider } from './CompletionProvider';
@@ -51,6 +52,7 @@ export class CompletionRouter implements SettingsConfigurable, Closeable {
         private readonly documentManager: DocumentManager,
         private readonly schemaRetriever: SchemaRetriever,
         private readonly entityFieldCompletionProviderMap = createEntityFieldProviders(),
+        private readonly usageTracker: UsageTracker,
     ) {}
 
     @Track({ name: 'getCompletions', trackObjectKey: 'items' })
@@ -95,6 +97,7 @@ export class CompletionRouter implements SettingsConfigurable, Closeable {
 
         if (completions instanceof Promise) {
             return await completions.then((result) => {
+                trackCompletion(this.usageTracker, provider, result);
                 return this.formatter.format(
                     {
                         isIncomplete: result.length > this.completionSettings.maxCompletions,
@@ -107,6 +110,7 @@ export class CompletionRouter implements SettingsConfigurable, Closeable {
                 );
             });
         } else if (completions) {
+            trackCompletion(this.usageTracker, provider, completions);
             const completionList = {
                 isIncomplete: completions.length > this.completionSettings.maxCompletions,
                 items: completions.slice(0, this.completionSettings.maxCompletions),
@@ -284,6 +288,7 @@ export class CompletionRouter implements SettingsConfigurable, Closeable {
             core.documentManager,
             external.schemaRetriever,
             createEntityFieldProviders(),
+            core.usageTracker,
         );
     }
 }
@@ -325,3 +330,13 @@ export function createEntityFieldProviders() {
     entityFieldProviderMap.set(EntityType.Output, new EntityFieldCompletionProvider<Output>());
     return entityFieldProviderMap;
 }
+
+function trackCompletion(
+    tracker: UsageTracker,
+    provider: CompletionProvider | undefined,
+    completions: CompletionItem[],
+) {
+    if (provider !== undefined && !(provider instanceof TopLevelSectionCompletionProvider) && completions.length > 0) {
+        tracker.track(EventType.MeaningfulCompletion);
+    }
+}

src/handlers/HoverHandler.ts

@@ -2,6 +2,7 @@ import { Hover, HoverParams, MarkupKind } from 'vscode-languageserver';
 import { ServerRequestHandler } from 'vscode-languageserver/lib/common/server';
 import { ServerComponents } from '../server/ServerComponents';
 import { TelemetryService } from '../telemetry/TelemetryService';
+import { EventType } from '../usageTracker/UsageTracker';
 
 export function hoverHandler(
     components: ServerComponents,
@@ -15,6 +16,7 @@ export function hoverHandler(
             };
         }
 
+        components.usageTracker.track(EventType.MeaningfulHover);
         return {
             contents: {
                 kind: MarkupKind.Markdown,

src/handlers/ResourceHandler.ts

@@ -22,6 +22,7 @@ import { ServerComponents } from '../server/ServerComponents';
 import { GetStackTemplateParams, GetStackTemplateResult } from '../stacks/StackRequestType';
 import { LoggerFactory } from '../telemetry/LoggerFactory';
 import { TelemetryService } from '../telemetry/TelemetryService';
+import { EventType } from '../usageTracker/UsageTracker';
 import { parseWithPrettyError } from '../utils/ZodErrorWrapper';
 
 const log = LoggerFactory.getLogger('ResourceHandler');
@@ -80,6 +81,7 @@ export function importResourceStateHandler(
     components: ServerComponents,
 ): ServerRequestHandler<ResourceStateParams, ResourceStateResult, never, void> {
     return async (params: ResourceStateParams): Promise<ResourceStateResult> => {
+        components.usageTracker.track(EventType.DidImportResources);
         return await components.resourceStateImporter.importResourceState(params);
     };
 }

src/handlers/StackHandler.ts

@@ -50,6 +50,7 @@ import {
     DescribeChangeSetResult,
 } from '../stacks/StackRequestType';
 import { TelemetryService } from '../telemetry/TelemetryService';
+import { EventType } from '../usageTracker/UsageTracker';
 import { handleLspError } from '../utils/Errors';
 import { parseWithPrettyError } from '../utils/ZodErrorWrapper';
 
@@ -111,6 +112,7 @@ export function createValidationHandler(
 ): RequestHandler<CreateValidationParams, CreateStackActionResult, void> {
     return async (rawParams) => {
         return await TelemetryService.instance.get('StackHandler').measureAsync('createValidation', async () => {
+            components.usageTracker.track(EventType.DidValidation);
             try {
                 const params = parseWithPrettyError(parseCreateValidationParams, rawParams);
                 return await components.validationWorkflowService.start(params);
@@ -126,6 +128,7 @@ export function createDeploymentHandler(
 ): RequestHandler<CreateDeploymentParams, CreateStackActionResult, void> {
     return async (rawParams) => {
         return await TelemetryService.instance.get('StackHandler').measureAsync('createDeployment', async () => {
+            components.usageTracker.track(EventType.DidDeployment);
             try {
                 const params = parseWithPrettyError(parseCreateDeploymentParams, rawParams);
                 return await components.deploymentWorkflowService.start(params);

src/server/CfnInfraCore.ts

@@ -11,6 +11,8 @@ import { SettingsManager } from '../settings/SettingsManager';
 import { ValidationManager } from '../stacks/actions/ValidationManager';
 import { ClientMessage } from '../telemetry/ClientMessage';
 import { TelemetryService } from '../telemetry/TelemetryService';
+import { UsageTracker } from '../usageTracker/UsageTracker';
+import { UsageTrackerMetrics } from '../usageTracker/UsageTrackerMetrics';
 import { Closeable, closeSafely } from '../utils/Closeable';
 import { Configurable, Configurables } from '../utils/Configurable';
 import { ExtendedInitializeParams } from './InitParams';
@@ -34,6 +36,8 @@ export class CfnInfraCore implements Configurables, Closeable {
     readonly validationManager: ValidationManager;
     readonly diagnosticCoordinator: DiagnosticCoordinator;
     readonly cloudformationEndpoint?: string;
+    readonly usageTracker: UsageTracker;
+    readonly usageTrackerMetrics: UsageTrackerMetrics;
 
     constructor(
         lspComponents: LspComponents,
@@ -68,6 +72,9 @@ export class CfnInfraCore implements Configurables, Closeable {
         this.diagnosticCoordinator =
             overrides.diagnosticCoordinator ??
             new DiagnosticCoordinator(lspComponents.diagnostics, this.syntaxTreeManager, this.validationManager);
+
+        this.usageTracker = overrides.usageTracker ?? new UsageTracker();
+        this.usageTrackerMetrics = overrides.usageTrackerMetrics ?? new UsageTrackerMetrics(this.usageTracker);
     }
 
     configurables(): Configurable[] {

src/usageTracker/UsageTracker.ts

@@ -0,0 +1,27 @@
+export enum EventType {
+    DidDeployment = 'DidDeployment',
+    DidValidation = 'DidValidation',
+    DidImportResources = 'DidImportResources',
+    MeaningfulHover = 'MeaningfulHover',
+    MeaningfulCompletion = 'MeaningfulCompletion',
+}
+
+export class UsageTracker {
+    private readonly events = new Set<EventType>();
+
+    track(event: EventType): void {
+        this.events.add(event);
+    }
+
+    allUsed(...events: EventType[]): boolean {
+        return events.every((event) => {
+            return this.events.has(event);
+        });
+    }
+
+    someUsed(...events: EventType[]): boolean {
+        return events.some((event) => {
+            return this.events.has(event);
+        });
+    }
+}

src/usageTracker/UsageTrackerMetrics.ts

@@ -0,0 +1,25 @@
+import { ScopedTelemetry } from '../telemetry/ScopedTelemetry';
+import { Telemetry } from '../telemetry/TelemetryDecorator';
+import { EventType, UsageTracker } from './UsageTracker';
+
+export class UsageTrackerMetrics {
+    @Telemetry() private readonly telemetry!: ScopedTelemetry;
+
+    constructor(private readonly usageTracker: UsageTracker) {
+        this.registerMetrics();
+    }
+
+    private registerMetrics(): void {
+        this.telemetry.registerGaugeProvider('cloudformation.used', () =>
+            this.usageTracker.someUsed(
+                EventType.DidDeployment,
+                EventType.DidValidation,
+                EventType.DidImportResources,
+                EventType.MeaningfulHover,
+                EventType.MeaningfulCompletion,
+            )
+                ? 1
+                : 0,
+        );
+    }
+}

tst/unit/autocomplete/CompletionRouter.test.ts

@@ -10,6 +10,7 @@ import { EntityType } from '../../../src/context/semantic/SemanticTypes';
 import { SyntaxTreeManager } from '../../../src/context/syntaxtree/SyntaxTreeManager';
 import { DocumentType } from '../../../src/document/Document';
 import { CombinedSchemas } from '../../../src/schema/CombinedSchemas';
+import { UsageTracker } from '../../../src/usageTracker/UsageTracker';
 import { ExtensionName } from '../../../src/utils/ExtensionConfig';
 import { createResourceContext, createTopLevelContext } from '../../utils/MockContext';
 import {
@@ -250,6 +251,7 @@ describe('CompletionRouter', () => {
             mockDocumentManager,
             mockComponents.schemaRetriever,
             entityFieldProviderMap,
+            new UsageTracker(),
         );
 
         function expectCompletionProvider(

tst/unit/server/UsageTracker.test.ts

@@ -0,0 +1,71 @@
+import { beforeEach, describe, expect, test } from 'vitest';
+import { EventType, UsageTracker } from '../../../src/usageTracker/UsageTracker';
+
+describe('UsageTracker', () => {
+    let tracker: UsageTracker;
+
+    beforeEach(() => {
+        tracker = new UsageTracker();
+    });
+
+    describe('track', () => {
+        test('tracks single event', () => {
+            tracker.track(EventType.DidDeployment);
+            expect(tracker.allUsed(EventType.DidDeployment)).toBe(true);
+        });
+
+        test('tracks multiple events', () => {
+            tracker.track(EventType.DidDeployment);
+            tracker.track(EventType.DidValidation);
+            expect(tracker.allUsed(EventType.DidDeployment, EventType.DidValidation)).toBe(true);
+        });
+
+        test('does not duplicate events', () => {
+            tracker.track(EventType.DidDeployment);
+            tracker.track(EventType.DidDeployment);
+            expect(tracker.allUsed(EventType.DidDeployment)).toBe(true);
+        });
+    });
+
+    describe('allUsed', () => {
+        test('returns true when all events tracked', () => {
+            tracker.track(EventType.DidDeployment);
+            tracker.track(EventType.DidValidation);
+            expect(tracker.allUsed(EventType.DidDeployment, EventType.DidValidation)).toBe(true);
+        });
+
+        test('returns false when some events not tracked', () => {
+            tracker.track(EventType.DidDeployment);
+            expect(tracker.allUsed(EventType.DidDeployment, EventType.DidValidation)).toBe(false);
+        });
+
+        test('returns false when no events tracked', () => {
+            expect(tracker.allUsed(EventType.DidDeployment)).toBe(false);
+        });
+
+        test('returns true for empty list', () => {
+            expect(tracker.allUsed()).toBe(true);
+        });
+    });
+
+    describe('someUsed', () => {
+        test('returns true when at least one event tracked', () => {
+            tracker.track(EventType.DidDeployment);
+            expect(tracker.someUsed(EventType.DidDeployment, EventType.DidValidation)).toBe(true);
+        });
+
+        test('returns false when no events tracked', () => {
+            expect(tracker.someUsed(EventType.DidDeployment, EventType.DidValidation)).toBe(false);
+        });
+
+        test('returns true when all events tracked', () => {
+            tracker.track(EventType.DidDeployment);
+            tracker.track(EventType.DidValidation);
+            expect(tracker.someUsed(EventType.DidDeployment, EventType.DidValidation)).toBe(true);
+        });
+
+        test('returns false for empty list', () => {
+            expect(tracker.someUsed()).toBe(false);
+        });
+    });
+});

tst/utils/MockServerComponents.ts

@@ -65,6 +65,8 @@ import { ValidationWorkflow } from '../../src/stacks/actions/ValidationWorkflow'
 import { StackEventManager } from '../../src/stacks/StackEventManager';
 import { StackManager } from '../../src/stacks/StackManager';
 import { ClientMessage } from '../../src/telemetry/ClientMessage';
+import { UsageTracker } from '../../src/usageTracker/UsageTracker';
+import { UsageTrackerMetrics } from '../../src/usageTracker/UsageTrackerMetrics';
 import { Closeable } from '../../src/utils/Closeable';
 import { Configurables } from '../../src/utils/Configurable';
 
@@ -368,6 +370,8 @@ export function createMockComponents(o: Partial<CfnLspServerComponentsType> = {}
         awsCredentials: overrides.awsCredentials ?? createMockAwsCredentials(),
         validationManager: overrides.validationManager ?? stubInterface<ValidationManager>(),
         diagnosticCoordinator: overrides.diagnosticCoordinator ?? createMockDiagnosticCoordinator(),
+        usageTracker: stubInterface<UsageTracker>(),
+        usageTrackerMetrics: stubInterface<UsageTrackerMetrics>(),
         close: () => Promise.resolve(),
         configurables: () => [],
     };