testing: add data for test failures (microsoft#209883)

* testing: add data for test failures

* fixup test

Author: connor4312

src/vs/workbench/contrib/testing/common/testResult.ts

@@ -12,6 +12,7 @@ import { IObservable, observableValue } from 'vs/base/common/observable';
 import { language } from 'vs/base/common/platform';
 import { WellDefinedPrefixTree } from 'vs/base/common/prefixTree';
 import { localize } from 'vs/nls';
+import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
 import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';
 import { IComputedStateAccessor, refreshComputedState } from 'vs/workbench/contrib/testing/common/getComputedState';
 import { TestCoverage } from 'vs/workbench/contrib/testing/common/testCoverage';
@@ -334,6 +335,7 @@ export class LiveTestResult extends Disposable implements ITestResult {
 		public readonly id: string,
 		public readonly persist: boolean,
 		public readonly request: ResolvedTestRunRequest,
+		@ITelemetryService private readonly telemetry: ITelemetryService,
 	) {
 		super();
 	}
@@ -481,6 +483,21 @@ export class LiveTestResult extends Disposable implements ITestResult {
 
 		this._completedAt = Date.now();
 		this.completeEmitter.fire();
+
+		this.telemetry.publicLog2<
+			{ failures: number; passes: number; controller: string },
+			{
+				owner: 'connor4312';
+				comment: 'Test outcome metrics. This helps us understand magnitude of feature use and how to build fix suggestions.';
+				failures: { comment: 'Number of test failures'; classification: 'SystemMetaData'; purpose: 'FeatureInsight' };
+				passes: { comment: 'Number of test failures'; classification: 'SystemMetaData'; purpose: 'FeatureInsight' };
+				controller: { comment: 'The test controller being used'; classification: 'SystemMetaData'; purpose: 'FeatureInsight' };
+			}
+		>('test.outcomes', {
+			failures: this.counts[TestResultState.Errored] + this.counts[TestResultState.Failed],
+			passes: this.counts[TestResultState.Passed],
+			controller: this.request.targets.map(t => t.controllerId).join(',')
+		});
 	}
 
 	/**

src/vs/workbench/contrib/testing/common/testResultService.ts

@@ -11,6 +11,7 @@ import { Disposable, DisposableStore, dispose, toDisposable } from 'vs/base/comm
 import { generateUuid } from 'vs/base/common/uuid';
 import { IContextKey, IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
 import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
+import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
 import { TestingContextKeys } from 'vs/workbench/contrib/testing/common/testingContextKeys';
 import { ITestProfileService } from 'vs/workbench/contrib/testing/common/testProfileService';
 import { ITestResult, LiveTestResult, TestResultItemChange, TestResultItemChangeReason } from 'vs/workbench/contrib/testing/common/testResult';
@@ -110,6 +111,7 @@ export class TestResultService extends Disposable implements ITestResultService
 		@IContextKeyService contextKeyService: IContextKeyService,
 		@ITestResultStorage private readonly storage: ITestResultStorage,
 		@ITestProfileService private readonly testProfiles: ITestProfileService,
+		@ITelemetryService private readonly telemetryService: ITelemetryService,
 	) {
 		super();
 		this._register(toDisposable(() => dispose(this._resultsDisposables)));
@@ -137,7 +139,7 @@ export class TestResultService extends Disposable implements ITestResultService
 	public createLiveResult(req: ResolvedTestRunRequest | ExtensionRunTestsRequest) {
 		if ('targets' in req) {
 			const id = generateUuid();
-			return this.push(new LiveTestResult(id, true, req));
+			return this.push(new LiveTestResult(id, true, req, this.telemetryService));
 		}
 
 		let profile: ITestRunProfile | undefined;
@@ -162,7 +164,7 @@ export class TestResultService extends Disposable implements ITestResultService
 			});
 		}
 
-		return this.push(new LiveTestResult(req.id, req.persist, resolved));
+		return this.push(new LiveTestResult(req.id, req.persist, resolved, this.telemetryService));
 	}
 
 	/**

src/vs/workbench/contrib/testing/common/testServiceImpl.ts

@@ -8,28 +8,28 @@ import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cance
 import { Emitter } from 'vs/base/common/event';
 import { Iterable } from 'vs/base/common/iterator';
 import { Disposable, DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
+import { isDefined } from 'vs/base/common/types';
 import { localize } from 'vs/nls';
+import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { IContextKey, IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { IStorageService, StorageScope, StorageTarget } from 'vs/platform/storage/common/storage';
+import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';
 import { IWorkspaceTrustRequestService } from 'vs/platform/workspace/common/workspaceTrust';
+import { getTestingConfiguration, TestingConfigKeys } from 'vs/workbench/contrib/testing/common/configuration';
 import { MainThreadTestCollection } from 'vs/workbench/contrib/testing/common/mainThreadTestCollection';
 import { MutableObservableValue } from 'vs/workbench/contrib/testing/common/observableValue';
 import { StoredValue } from 'vs/workbench/contrib/testing/common/storedValue';
-import { ResolvedTestRunRequest, TestDiffOpType, TestsDiff } from 'vs/workbench/contrib/testing/common/testTypes';
 import { TestExclusions } from 'vs/workbench/contrib/testing/common/testExclusions';
 import { TestId } from 'vs/workbench/contrib/testing/common/testId';
 import { TestingContextKeys } from 'vs/workbench/contrib/testing/common/testingContextKeys';
 import { canUseProfileWithTest, ITestProfileService } from 'vs/workbench/contrib/testing/common/testProfileService';
 import { ITestResult } from 'vs/workbench/contrib/testing/common/testResult';
 import { ITestResultService } from 'vs/workbench/contrib/testing/common/testResultService';
 import { AmbiguousRunTestsRequest, IMainThreadTestController, ITestService } from 'vs/workbench/contrib/testing/common/testService';
+import { ResolvedTestRunRequest, TestDiffOpType, TestsDiff } from 'vs/workbench/contrib/testing/common/testTypes';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
-import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
-import { getTestingConfiguration, TestingConfigKeys } from 'vs/workbench/contrib/testing/common/configuration';
-import { isDefined } from 'vs/base/common/types';
-import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';
 
 export class TestService extends Disposable implements ITestService {
 	declare readonly _serviceBrand: undefined;

src/vs/workbench/contrib/testing/test/common/testResultService.test.ts

@@ -10,6 +10,7 @@ import { CancellationTokenSource } from 'vs/base/common/cancellation';
 import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils';
 import { MockContextKeyService } from 'vs/platform/keybinding/test/common/mockKeybindingService';
 import { NullLogService } from 'vs/platform/log/common/log';
+import { NullTelemetryService } from 'vs/platform/telemetry/common/telemetryUtils';
 import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';
 import { TestId } from 'vs/workbench/contrib/testing/common/testId';
 import { TestProfileService } from 'vs/workbench/contrib/testing/common/testProfileService';
@@ -45,7 +46,7 @@ suite('Workbench - Test Results Service', () => {
 			persist: boolean,
 			request: ResolvedTestRunRequest,
 		) {
-			super(id, persist, request);
+			super(id, persist, request, NullTelemetryService);
 			ds.add(this);
 		}
 
@@ -217,6 +218,7 @@ suite('Workbench - Test Results Service', () => {
 				new MockContextKeyService(),
 				storage,
 				ds.add(new TestProfileService(new MockContextKeyService(), ds.add(new TestStorageService()))),
+				NullTelemetryService,
 			));
 		});
 
@@ -235,6 +237,7 @@ suite('Workbench - Test Results Service', () => {
 				new MockContextKeyService(),
 				storage,
 				ds.add(new TestProfileService(new MockContextKeyService(), ds.add(new TestStorageService()))),
+				NullTelemetryService,
 			));
 
 			assert.strictEqual(0, results.results.length);
@@ -260,6 +263,7 @@ suite('Workbench - Test Results Service', () => {
 				'',
 				false,
 				defaultOpts([]),
+				NullTelemetryService,
 			));
 			results.clear();
 
@@ -272,6 +276,7 @@ suite('Workbench - Test Results Service', () => {
 				'',
 				false,
 				defaultOpts([]),
+				NullTelemetryService,
 			));
 
 			assert.deepStrictEqual(results.results, [r2, r]);

src/vs/workbench/contrib/testing/test/common/testResultStorage.test.ts

@@ -8,6 +8,7 @@ import { range } from 'vs/base/common/arrays';
 import { DisposableStore } from 'vs/base/common/lifecycle';
 import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils';
 import { NullLogService } from 'vs/platform/log/common/log';
+import { NullTelemetryService } from 'vs/platform/telemetry/common/telemetryUtils';
 import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';
 import { ITestResult, LiveTestResult } from 'vs/workbench/contrib/testing/common/testResult';
 import { InMemoryResultStorage, RETAIN_MAX_RESULTS } from 'vs/workbench/contrib/testing/common/testResultStorage';
@@ -22,7 +23,8 @@ suite('Workbench - Test Result Storage', () => {
 		const t = ds.add(new LiveTestResult(
 			'',
 			true,
-			{ targets: [] }
+			{ targets: [] },
+			NullTelemetryService,
 		));
 
 		t.addTask({ id: taskName, name: undefined, running: true });