speech - scaffold keyword activation (microsoft#202643)

* speech - scaffold keyword activation

* first cut settting for keyword activation

* handle lifecycle better

* .

* .

* .

* tweaks

* show a status bar entry

* add in context option

* cleanup

Author: bpasero

src/vs/base/common/event.ts

@@ -622,27 +622,6 @@ export namespace Event {
 		return event(e => handler(e));
 	}
 
-	/**
-	 * Adds a listener to an event and calls the listener immediately with undefined as the event object. A new
-	 * {@link DisposableStore} is passed to the listener which is disposed when the returned disposable is disposed.
-	 */
-	export function runAndSubscribeWithStore<T>(event: Event<T>, handler: (e: T | undefined, disposableStore: DisposableStore) => any): IDisposable {
-		let store: DisposableStore | null = null;
-
-		function run(e: T | undefined) {
-			store?.dispose();
-			store = new DisposableStore();
-			handler(e, store);
-		}
-
-		run(undefined);
-		const disposable = event(e => run(e));
-		return toDisposable(() => {
-			disposable.dispose();
-			store?.dispose();
-		});
-	}
-
 	class EmitterObserver<T> implements IObserver {
 
 		readonly emitter: Emitter<T>;

src/vs/base/test/common/event.test.ts

@@ -8,7 +8,7 @@ import { DeferredPromise, timeout } from 'vs/base/common/async';
 import { CancellationToken } from 'vs/base/common/cancellation';
 import { errorHandler, setUnexpectedErrorHandler } from 'vs/base/common/errors';
 import { AsyncEmitter, DebounceEmitter, DynamicListEventMultiplexer, Emitter, Event, EventBufferer, EventMultiplexer, IWaitUntil, MicrotaskEmitter, PauseableEmitter, Relay, createEventDeliveryQueue } from 'vs/base/common/event';
-import { DisposableStore, IDisposable, isDisposable, setDisposableTracker, toDisposable, DisposableTracker } from 'vs/base/common/lifecycle';
+import { DisposableStore, IDisposable, isDisposable, setDisposableTracker, DisposableTracker } from 'vs/base/common/lifecycle';
 import { observableValue, transaction } from 'vs/base/common/observable';
 import { MicrotaskDelay } from 'vs/base/common/symbols';
 import { runWithFakedTimers } from 'vs/base/test/common/timeTravelScheduler';
@@ -1272,36 +1272,6 @@ suite('Event utils', () => {
 		});
 	});
 
-	test('runAndSubscribeWithStore', () => {
-		const eventEmitter = ds.add(new Emitter());
-		const event = eventEmitter.event;
-
-		let i = 0;
-		const log = new Array<any>();
-		const disposable = Event.runAndSubscribeWithStore(event, (e, disposables) => {
-			const idx = i++;
-			log.push({ label: 'handleEvent', data: e || null, idx });
-			disposables.add(toDisposable(() => {
-				log.push({ label: 'dispose', idx });
-			}));
-		});
-
-		log.push({ label: 'fire' });
-		eventEmitter.fire('someEventData');
-
-		log.push({ label: 'disposeAll' });
-		disposable.dispose();
-
-		assert.deepStrictEqual(log, [
-			{ label: 'handleEvent', data: null, idx: 0 },
-			{ label: 'fire' },
-			{ label: 'dispose', idx: 0 },
-			{ label: 'handleEvent', data: 'someEventData', idx: 1 },
-			{ label: 'disposeAll' },
-			{ label: 'dispose', idx: 1 },
-		]);
-	});
-
 	suite('accumulate', () => {
 		test('should not fire after a listener is disposed with undefined or []', async () => {
 			const eventEmitter = ds.add(new Emitter<number>());

src/vs/workbench/api/browser/mainThreadSpeech.ts

@@ -8,20 +8,26 @@ import { Emitter } from 'vs/base/common/event';
 import { Disposable, DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
 import { ILogService } from 'vs/platform/log/common/log';
 import { ExtHostContext, ExtHostSpeechShape, MainContext, MainThreadSpeechShape } from 'vs/workbench/api/common/extHost.protocol';
-import { ISpeechProviderMetadata, ISpeechService, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService';
+import { IKeywordRecognitionEvent, ISpeechProviderMetadata, ISpeechService, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService';
 import { IExtHostContext, extHostNamedCustomer } from 'vs/workbench/services/extensions/common/extHostCustomers';
 
 type SpeechToTextSession = {
 	readonly onDidChange: Emitter<ISpeechToTextEvent>;
 };
 
+type KeywordRecognitionSession = {
+	readonly onDidChange: Emitter<IKeywordRecognitionEvent>;
+};
+
 @extHostNamedCustomer(MainContext.MainThreadSpeech)
 export class MainThreadSpeech extends Disposable implements MainThreadSpeechShape {
 
 	private readonly proxy: ExtHostSpeechShape;
 
 	private readonly providerRegistrations = new Map<number, IDisposable>();
-	private readonly providerSessions = new Map<number, SpeechToTextSession>();
+
+	private readonly speechToTextSessions = new Map<number, SpeechToTextSession>();
+	private readonly keywordRecognitionSessions = new Map<number, KeywordRecognitionSession>();
 
 	constructor(
 		extHostContext: IExtHostContext,
@@ -47,13 +53,33 @@ export class MainThreadSpeech extends Disposable implements MainThreadSpeechShap
 				disposables.add(token.onCancellationRequested(() => this.proxy.$cancelSpeechToTextSession(session)));
 
 				const onDidChange = disposables.add(new Emitter<ISpeechToTextEvent>());
-				this.providerSessions.set(session, { onDidChange });
+				this.speechToTextSessions.set(session, { onDidChange });
 
 				return {
 					onDidChange: onDidChange.event,
 					dispose: () => {
 						cts.dispose(true);
-						this.providerSessions.delete(session);
+						this.speechToTextSessions.delete(session);
+						disposables.dispose();
+					}
+				};
+			},
+			createKeywordRecognitionSession: token => {
+				const disposables = new DisposableStore();
+				const cts = new CancellationTokenSource(token);
+				const session = Math.random();
+
+				this.proxy.$createKeywordRecognitionSession(handle, session);
+				disposables.add(token.onCancellationRequested(() => this.proxy.$cancelKeywordRecognitionSession(session)));
+
+				const onDidChange = disposables.add(new Emitter<IKeywordRecognitionEvent>());
+				this.keywordRecognitionSessions.set(session, { onDidChange });
+
+				return {
+					onDidChange: onDidChange.event,
+					dispose: () => {
+						cts.dispose(true);
+						this.keywordRecognitionSessions.delete(session);
 						disposables.dispose();
 					}
 				};
@@ -75,9 +101,12 @@ export class MainThreadSpeech extends Disposable implements MainThreadSpeechShap
 	}
 
 	$emitSpeechToTextEvent(session: number, event: ISpeechToTextEvent): void {
-		const providerSession = this.providerSessions.get(session);
-		if (providerSession) {
-			providerSession.onDidChange.fire(event);
-		}
+		const providerSession = this.speechToTextSessions.get(session);
+		providerSession?.onDidChange.fire(event);
+	}
+
+	$emitKeywordRecognitionEvent(session: number, event: IKeywordRecognitionEvent): void {
+		const providerSession = this.keywordRecognitionSessions.get(session);
+		providerSession?.onDidChange.fire(event);
 	}
 }

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -1627,7 +1627,8 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			StackFrameFocus: extHostTypes.StackFrameFocus,
 			ThreadFocus: extHostTypes.ThreadFocus,
 			RelatedInformationType: extHostTypes.RelatedInformationType,
-			SpeechToTextStatus: extHostTypes.SpeechToTextStatus
+			SpeechToTextStatus: extHostTypes.SpeechToTextStatus,
+			KeywordRecognitionStatus: extHostTypes.KeywordRecognitionStatus
 		};
 	};
 }

src/vs/workbench/api/common/extHost.protocol.ts

@@ -64,7 +64,7 @@ import { ICellRange } from 'vs/workbench/contrib/notebook/common/notebookRange';
 import { InputValidationType } from 'vs/workbench/contrib/scm/common/scm';
 import { IWorkspaceSymbol, NotebookPriorityInfo } from 'vs/workbench/contrib/search/common/search';
 import { IRawClosedNotebookFileMatch } from 'vs/workbench/contrib/search/common/searchNotebookHelpers';
-import { ISpeechProviderMetadata, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService';
+import { IKeywordRecognitionEvent, ISpeechProviderMetadata, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService';
 import { CoverageDetails, ExtensionRunTestsRequest, ICallProfileRunHandler, IFileCoverage, ISerializedTestResults, IStartControllerTests, ITestItem, ITestMessage, ITestRunProfile, ITestRunTask, ResolvedTestRunRequest, TestResultState, TestsDiffOp } from 'vs/workbench/contrib/testing/common/testTypes';
 import { Timeline, TimelineChangeEvent, TimelineOptions, TimelineProviderDescriptor } from 'vs/workbench/contrib/timeline/common/timeline';
 import { TypeHierarchyItem } from 'vs/workbench/contrib/typeHierarchy/common/typeHierarchy';
@@ -1159,11 +1159,15 @@ export interface MainThreadSpeechShape extends IDisposable {
 	$unregisterProvider(handle: number): void;
 
 	$emitSpeechToTextEvent(session: number, event: ISpeechToTextEvent): void;
+	$emitKeywordRecognitionEvent(session: number, event: IKeywordRecognitionEvent): void;
 }
 
 export interface ExtHostSpeechShape {
 	$createSpeechToTextSession(handle: number, session: number): Promise<void>;
 	$cancelSpeechToTextSession(session: number): Promise<void>;
+
+	$createKeywordRecognitionSession(handle: number, session: number): Promise<void>;
+	$cancelKeywordRecognitionSession(session: number): Promise<void>;
 }
 
 export interface MainThreadChatProviderShape extends IDisposable {

src/vs/workbench/api/common/extHostSpeech.ts

@@ -52,6 +52,34 @@ export class ExtHostSpeech implements ExtHostSpeechShape {
 		this.sessions.delete(session);
 	}
 
+	async $createKeywordRecognitionSession(handle: number, session: number): Promise<void> {
+		const provider = this.providers.get(handle);
+		if (!provider) {
+			return;
+		}
+
+		const disposables = new DisposableStore();
+
+		const cts = new CancellationTokenSource();
+		this.sessions.set(session, cts);
+
+		const keywordRecognitionSession = disposables.add(provider.provideKeywordRecognitionSession(cts.token));
+		disposables.add(keywordRecognitionSession.onDidChange(e => {
+			if (cts.token.isCancellationRequested) {
+				return;
+			}
+
+			this.proxy.$emitKeywordRecognitionEvent(session, e);
+		}));
+
+		disposables.add(cts.token.onCancellationRequested(() => disposables.dispose()));
+	}
+
+	async $cancelKeywordRecognitionSession(session: number): Promise<void> {
+		this.sessions.get(session)?.dispose(true);
+		this.sessions.delete(session);
+	}
+
 	registerProvider(extension: ExtensionIdentifier, identifier: string, provider: vscode.SpeechProvider): IDisposable {
 		const handle = ExtHostSpeech.ID_POOL++;
 

src/vs/workbench/api/common/extHostTypes.ts

@@ -4205,4 +4205,9 @@ export enum SpeechToTextStatus {
 	Stopped = 4
 }
 
+export enum KeywordRecognitionStatus {
+	Recognized = 1,
+	Stopped = 2
+}
+
 //#endregion

src/vs/workbench/contrib/accessibility/browser/accessibility.contribution.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions';
-import { registerAccessibilityConfiguration } from 'vs/workbench/contrib/accessibility/browser/accessibilityConfiguration';
+import { DynamicSpeechAccessibilityConfiguration, registerAccessibilityConfiguration } from 'vs/workbench/contrib/accessibility/browser/accessibilityConfiguration';
 import { IWorkbenchContributionsRegistry, Extensions as WorkbenchExtensions } from 'vs/workbench/common/contributions';
 import { LifecyclePhase } from 'vs/workbench/services/lifecycle/common/lifecycle';
 import { Registry } from 'vs/platform/registry/common/platform';
@@ -30,3 +30,4 @@ workbenchContributionsRegistry.registerWorkbenchContribution(NotificationAccessi
 workbenchContributionsRegistry.registerWorkbenchContribution(InlineCompletionsAccessibleViewContribution, LifecyclePhase.Eventually);
 workbenchContributionsRegistry.registerWorkbenchContribution(AccessibilityStatus, LifecyclePhase.Ready);
 workbenchContributionsRegistry.registerWorkbenchContribution(SaveAudioCueContribution, LifecyclePhase.Ready);
+workbenchContributionsRegistry.registerWorkbenchContribution(DynamicSpeechAccessibilityConfiguration, LifecyclePhase.Ready);

src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts

@@ -9,6 +9,10 @@ import { Registry } from 'vs/platform/registry/common/platform';
 import { RawContextKey } from 'vs/platform/contextkey/common/contextkey';
 import { workbenchConfigurationNodeBase } from 'vs/workbench/common/configuration';
 import { AccessibilityAlertSettingId } from 'vs/platform/audioCues/browser/audioCueService';
+import { ISpeechService } from 'vs/workbench/contrib/speech/common/speechService';
+import { Disposable } from 'vs/base/common/lifecycle';
+import { IWorkbenchContribution } from 'vs/workbench/common/contributions';
+import { Event } from 'vs/base/common/event';
 
 export const accessibilityHelpIsShown = new RawContextKey<boolean>('accessibilityHelpIsShown', false, true);
 export const accessibleViewIsShown = new RawContextKey<boolean>('accessibleViewIsShown', false, true);
@@ -35,11 +39,6 @@ export const enum ViewDimUnfocusedOpacityProperties {
 	Maximum = 1
 }
 
-export const enum AccessibilityVoiceSettingId {
-	SpeechTimeout = 'accessibility.voice.speechTimeout',
-}
-export const SpeechTimeoutDefault = 1200;
-
 export const enum AccessibilityVerbositySettingId {
 	Terminal = 'accessibility.verbosity.terminal',
 	DiffEditor = 'accessibility.verbosity.diffEditor',
@@ -77,10 +76,14 @@ const baseProperty: object = {
 	tags: ['accessibility']
 };
 
-const configuration: IConfigurationNode = {
+export const accessibilityConfigurationNodeBase = Object.freeze<IConfigurationNode>({
 	id: 'accessibility',
 	title: localize('accessibilityConfigurationTitle', "Accessibility"),
-	type: 'object',
+	type: 'object'
+});
+
+const configuration: IConfigurationNode = {
+	...accessibilityConfigurationNodeBase,
 	properties: {
 		[AccessibilityVerbositySettingId.Terminal]: {
 			description: localize('verbosity.terminal.description', 'Provide information about how to access the terminal accessibility help menu when the terminal is focused.'),
@@ -251,13 +254,6 @@ const configuration: IConfigurationNode = {
 			'default': true,
 			tags: ['accessibility']
 		},
-		[AccessibilityVoiceSettingId.SpeechTimeout]: {
-			'markdownDescription': localize('voice.speechTimeout', "The duration in milliseconds that voice speech recognition remains active after you stop speaking. For example in a chat session, the transcribed text is submitted automatically after the timeout is met. Set to `0` to disable this feature."),
-			'type': 'number',
-			'default': SpeechTimeoutDefault,
-			'minimum': 0,
-			'tags': ['accessibility']
-		},
 		[AccessibilityWorkbenchSettingId.AccessibleViewCloseOnKeyPress]: {
 			markdownDescription: localize('terminal.integrated.accessibleView.closeOnKeyPress', "On keypress, close the Accessible View and focus the element from which it was invoked."),
 			type: 'boolean',
@@ -298,3 +294,39 @@ export function registerAccessibilityConfiguration() {
 		}
 	});
 }
+
+export const enum AccessibilityVoiceSettingId {
+	SpeechTimeout = 'accessibility.voice.speechTimeout'
+}
+export const SpeechTimeoutDefault = 1200;
+
+export class DynamicSpeechAccessibilityConfiguration extends Disposable implements IWorkbenchContribution {
+
+	constructor(
+		@ISpeechService private readonly speechService: ISpeechService
+	) {
+		super();
+
+		this._register(Event.runAndSubscribe(speechService.onDidRegisterSpeechProvider, () => this.updateConfiguration()));
+	}
+
+	private updateConfiguration(): void {
+		if (!this.speechService.hasSpeechProvider) {
+			return; // these settings require a speech provider
+		}
+
+		const registry = Registry.as<IConfigurationRegistry>(Extensions.Configuration);
+		registry.registerConfiguration({
+			...accessibilityConfigurationNodeBase,
+			properties: {
+				[AccessibilityVoiceSettingId.SpeechTimeout]: {
+					'markdownDescription': localize('voice.speechTimeout', "The duration in milliseconds that voice speech recognition remains active after you stop speaking. For example in a chat session, the transcribed text is submitted automatically after the timeout is met. Set to `0` to disable this feature."),
+					'type': 'number',
+					'default': SpeechTimeoutDefault,
+					'minimum': 0,
+					'tags': ['accessibility']
+				}
+			}
+		});
+	}
+}