Update telemetry events and exp allocations (microsoft#248035)

Author: bhavyaus

src/vs/workbench/contrib/welcomeGettingStarted/browser/gettingStarted.ts

@@ -73,6 +73,7 @@ import { AccessibilityVerbositySettingId } from '../../accessibility/browser/acc
 import { AccessibleViewAction } from '../../accessibility/browser/accessibleViewActions.js';
 import { KeybindingLabel } from '../../../../base/browser/ui/keybindingLabel/keybindingLabel.js';
 import { ScrollbarVisibility } from '../../../../base/common/scrollable.js';
+import { IGettingStartedExperimentService } from './gettingStartedExpService.js';
 
 const SLIDE_TRANSITION_TIME_MS = 250;
 const configurationKey = 'workbench.startupEditor';
@@ -190,7 +191,8 @@ export class GettingStartedPage extends EditorPane {
 		@IHostService private readonly hostService: IHostService,
 		@IWebviewService private readonly webviewService: IWebviewService,
 		@IWorkspaceContextService private readonly workspaceContextService: IWorkspaceContextService,
-		@IAccessibilityService private readonly accessibilityService: IAccessibilityService
+		@IAccessibilityService private readonly accessibilityService: IAccessibilityService,
+		@IGettingStartedExperimentService private readonly gettingStartedExperimentService: IGettingStartedExperimentService,
 	) {
 
 		super(GettingStartedPage.ID, group, telemetryService, themeService, storageService);
@@ -959,13 +961,18 @@ export class GettingStartedPage extends EditorPane {
 			const fistContentBehaviour = daysSinceFirstSession < 1 ? 'openToFirstCategory' : 'index';
 
 			if (fistContentBehaviour === 'openToFirstCategory') {
-				const first = this.gettingStartedCategories.filter(c => !c.when || this.contextService.contextMatchesRules(c.when))[0];
+				const exp = this.gettingStartedExperimentService.getCurrentExperiment();
+				const first = exp?.walkthroughId ? this.gettingStartedService.getWalkthrough(exp.walkthroughId) : this.gettingStartedCategories.filter(c => !c.when || this.contextService.contextMatchesRules(c.when))[0];
 				if (first) {
 					this.hasScrolledToFirstCategory = true;
 					this.currentWalkthrough = first;
 					this.editorInput.selectedCategory = this.currentWalkthrough?.id;
 					this.editorInput.walkthroughPageTitle = this.currentWalkthrough.walkthroughPageTitle;
-					this.buildCategorySlide(this.editorInput.selectedCategory, undefined);
+					if (first.id === NEW_WELCOME_EXPERIENCE) {
+						this.buildNewCategorySlide(this.editorInput.selectedCategory, undefined);
+					} else {
+						this.buildCategorySlide(this.editorInput.selectedCategory, undefined);
+					}
 					this.setSlide('details', true /* firstLaunch */);
 					return;
 				}
@@ -1410,6 +1417,7 @@ export class GettingStartedPage extends EditorPane {
 
 
 	private selectStepByIndex(newIndex: number, steps: IResolvedWalkthroughStep[], direction: number) {
+		this.telemetryService.publicLog2<GettingStartedActionEvent, GettingStartedActionClassification>('gettingStarted.ActionExecuted', { command: 'selectTask', argument: steps[newIndex].id, walkthroughId: this.currentWalkthrough?.id });
 		const currentIndex = steps.findIndex(step => step.id === this.editorInput.selectedStep);
 
 		// Update the selected step and build its media
@@ -1756,6 +1764,7 @@ export class GettingStartedPage extends EditorPane {
 	}
 
 	private selectSubStep(selectedStepId: string) {
+		this.telemetryService.publicLog2<GettingStartedActionEvent, GettingStartedActionClassification>('gettingStarted.ActionExecuted', { command: 'selectTask', argument: selectedStepId, walkthroughId: this.currentWalkthrough?.id });
 		if (this.editorInput.selectedStep === selectedStepId) {
 			return;
 		}

src/vs/workbench/contrib/welcomeGettingStarted/browser/gettingStartedExpService.ts

@@ -13,13 +13,15 @@ import { IProductService } from '../../../../platform/product/common/productServ
 export interface IGettingStartedExperiment {
 	cohort: number;
 	experimentGroup: string;
+	walkthroughId: string;
+	iteration: number;
 }
 
 export const IGettingStartedExperimentService = createDecorator<IGettingStartedExperimentService>('gettingStartedExperimentService');
 
 export interface IGettingStartedExperimentService {
 	readonly _serviceBrand: undefined;
-	getCurrentExperiment(): IGettingStartedExperiment;
+	getCurrentExperiment(): IGettingStartedExperiment | undefined;
 }
 
 const EXPERIMENT_STORAGE_KEY = 'gettingStartedExperiment';
@@ -28,33 +30,34 @@ interface ExperimentGroupDefinition {
 	name: string;
 	min: number;
 	max: number;
+	iteration: number;
+	walkthroughId: string;
 }
 
 export enum GettingStartedExperimentGroup {
 	New = 'newExp',
+	Control = 'controlExp',
 	Default = 'defaultExp'
 }
 
 const STABLE_EXPERIMENT_GROUPS: ExperimentGroupDefinition[] = [
-	// { name: GettingStartedExperimentGroup.New, min: 0.0, max: 0.1 },
-	//{ name: GettingStartedExperimentGroup.Default, min: 0.1, max: 1.0 }
-	{ name: GettingStartedExperimentGroup.Default, min: 0.0, max: 1.0 }
+	// Bump the iteration each time we change group allocations
+	{ name: GettingStartedExperimentGroup.New, min: 0.0, max: 0.1, iteration: 1, walkthroughId: 'NewWelcomeExperience' },
+	{ name: GettingStartedExperimentGroup.Control, min: 0.1, max: 0.2, iteration: 1, walkthroughId: 'Setup' },
+	{ name: GettingStartedExperimentGroup.Default, min: 0.2, max: 1, iteration: 1, walkthroughId: 'Setup' }
 ];
 
 const INSIDERS_EXPERIMENT_GROUPS: ExperimentGroupDefinition[] = [
-	// { name: GettingStartedExperimentGroup.New, min: 0.0, max: 0.3 },
-	//{ name: GettingStartedExperimentGroup.Default, min: 0.3, max: 1.0 }
-	{ name: GettingStartedExperimentGroup.Default, min: 0.0, max: 1.0 }
-];
-
-const DEFAULT_EXPERIMENT_GROUPS: ExperimentGroupDefinition[] = [
-	{ name: GettingStartedExperimentGroup.Default, min: 0.0, max: 1.0 }
+	// Bump the iteration each time we change group allocations
+	{ name: GettingStartedExperimentGroup.New, min: 0.0, max: 0.3, iteration: 1, walkthroughId: 'NewWelcomeExperience' },
+	{ name: GettingStartedExperimentGroup.Control, min: 0.3, max: 0.6, iteration: 1, walkthroughId: 'Setup' },
+	{ name: GettingStartedExperimentGroup.Default, min: 0.6, max: 1, iteration: 1, walkthroughId: 'Setup' }
 ];
 
 export class GettingStartedExperimentService extends Disposable implements IGettingStartedExperimentService {
 	declare readonly _serviceBrand: undefined;
 
-	private readonly experiment: IGettingStartedExperiment;
+	private readonly experiment: IGettingStartedExperiment | undefined;
 
 	constructor(
 		@IStorageService private readonly storageService: IStorageService,
@@ -66,18 +69,17 @@ export class GettingStartedExperimentService extends Disposable implements IGett
 		this.sendExperimentTelemetry();
 	}
 
-	private getExperimentAllocation(): ExperimentGroupDefinition[] {
+	private getExperimentAllocation(): ExperimentGroupDefinition[] | undefined {
 		const quality = this.productService.quality;
 		if (quality === 'stable') {
 			return STABLE_EXPERIMENT_GROUPS;
 		} else if (quality === 'insider') {
 			return INSIDERS_EXPERIMENT_GROUPS;
-		} else {
-			return DEFAULT_EXPERIMENT_GROUPS;
 		}
+		return;
 	}
 
-	private getOrCreateExperiment(): IGettingStartedExperiment {
+	private getOrCreateExperiment(): IGettingStartedExperiment | undefined {
 		const storedExperiment = this.storageService.get(EXPERIMENT_STORAGE_KEY, StorageScope.APPLICATION);
 		if (storedExperiment) {
 			try {
@@ -88,6 +90,9 @@ export class GettingStartedExperimentService extends Disposable implements IGett
 		}
 
 		const newExperiment = this.createNewExperiment();
+		if (!newExperiment) {
+			return;
+		}
 
 		this.storageService.store(
 			EXPERIMENT_STORAGE_KEY,
@@ -99,44 +104,55 @@ export class GettingStartedExperimentService extends Disposable implements IGett
 		return newExperiment;
 	}
 
-	private createNewExperiment(): IGettingStartedExperiment {
+	private createNewExperiment(): IGettingStartedExperiment | undefined {
 		const cohort = Math.random();
 		const experimentGroups = this.getExperimentAllocation();
+		if (!experimentGroups) {
+			return;
+		}
 
-		let experimentGroup = GettingStartedExperimentGroup.Default;
 		for (const group of experimentGroups) {
 			if (cohort >= group.min && cohort < group.max) {
-				experimentGroup = group.name as GettingStartedExperimentGroup;
-				break;
+				return { cohort, experimentGroup: group.name, walkthroughId: group.walkthroughId, iteration: group.iteration };
 			}
 		}
 
-		return { cohort, experimentGroup };
+		return;
 	}
 
 	private sendExperimentTelemetry(): void {
+		if (!this.experiment) {
+			return;
+		}
+
 		type GettingStartedExperimentClassification = {
 			owner: 'bhavyaus';
 			comment: 'Records which experiment cohort the user is in for getting started experience';
 			cohort: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The exact cohort number for the user' };
 			experimentGroup: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The experiment group the user is in' };
+			iteration: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The iteration number for the experiment' };
+			walkthroughId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The walkthrough ID for the experiment' };
 		};
 
 		type GettingStartedExperimentEvent = {
 			cohort: number;
 			experimentGroup: string;
+			iteration: number;
+			walkthroughId: string;
 		};
 
 		this.telemetryService.publicLog2<GettingStartedExperimentEvent, GettingStartedExperimentClassification>(
 			'gettingStarted.experimentCohort',
 			{
 				cohort: this.experiment.cohort,
-				experimentGroup: this.experiment.experimentGroup
+				experimentGroup: this.experiment.experimentGroup,
+				iteration: this.experiment.iteration,
+				walkthroughId: this.experiment.walkthroughId
 			}
 		);
 	}
 
-	getCurrentExperiment(): IGettingStartedExperiment {
+	getCurrentExperiment(): IGettingStartedExperiment | undefined {
 		return this.experiment;
 	}
 }