Skip to content

Commit 99df9dd

Browse files
TyriarTyriar
authored
Merge pull request microsoft#255225 from microsoft/merogge/bring-back-voice
bring back terminal voice input
2 parents ca0d33d + 5d7df7a commit 99df9dd

File tree

5 files changed

+260
-1
lines changed

5 files changed

+260
-1
lines changed

‎src/vs/workbench/contrib/terminal/browser/terminalActions.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ const category = terminalStrings.actionCategory;
6969

7070
// Some terminal context keys get complicated. Since normalizing and/or context keys can be
7171
// expensive this is done once per context key and shared.
72-
const sharedWhenClause = (() => {
72+
export const sharedWhenClause = (() => {
7373
const terminalAvailable = ContextKeyExpr.or(TerminalContextKeys.processSupported, TerminalContextKeys.terminalHasBeenCreated);
7474
return {
7575
terminalAvailable,

‎src/vs/workbench/contrib/terminal/terminal.all.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ import '../terminalContrib/sendSignal/browser/terminal.sendSignal.contribution.j
3333
import '../terminalContrib/suggest/browser/terminal.suggest.contribution.js';
3434
import '../terminalContrib/chat/browser/terminal.initialHint.contribution.js';
3535
import '../terminalContrib/wslRecommendation/browser/terminal.wslRecommendation.contribution.js';
36+
import '../terminalContrib/voice/browser/terminal.voice.contribution.js';

‎src/vs/workbench/contrib/terminalContrib/voice/browser/terminal.voice.contribution.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { registerTerminalVoiceActions } from './terminalVoiceActions.js';
7+
8+
registerTerminalVoiceActions();

‎src/vs/workbench/contrib/terminalContrib/voice/browser/terminalVoice.ts

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { RunOnceScheduler } from '../../../../../base/common/async.js';
7+
import { CancellationTokenSource } from '../../../../../base/common/cancellation.js';
8+
import { Codicon } from '../../../../../base/common/codicons.js';
9+
import { Disposable, DisposableStore, toDisposable } from '../../../../../base/common/lifecycle.js';
10+
import { ThemeIcon } from '../../../../../base/common/themables.js';
11+
import { isNumber } from '../../../../../base/common/types.js';
12+
import { localize } from '../../../../../nls.js';
13+
import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js';
14+
import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
15+
import { SpeechTimeoutDefault } from '../../../accessibility/browser/accessibilityConfiguration.js';
16+
import { ISpeechService, AccessibilityVoiceSettingId, ISpeechToTextEvent, SpeechToTextStatus } from '../../../speech/common/speechService.js';
17+
import type { IMarker, IDecoration } from '@xterm/xterm';
18+
import { alert } from '../../../../../base/browser/ui/aria/aria.js';
19+
import { ITerminalService } from '../../../terminal/browser/terminal.js';
20+
21+
22+
const symbolMap: { [key: string]: string } = {
23+
'Ampersand': '&',
24+
'ampersand': '&',
25+
'Dollar': '$',
26+
'dollar': '$',
27+
'Percent': '%',
28+
'percent': '%',
29+
'Asterisk': '*',
30+
'asterisk': '*',
31+
'Plus': '+',
32+
'plus': '+',
33+
'Equals': '=',
34+
'equals': '=',
35+
'Exclamation': '!',
36+
'exclamation': '!',
37+
'Slash': '/',
38+
'slash': '/',
39+
'Backslash': '\\',
40+
'backslash': '\\',
41+
'Dot': '.',
42+
'dot': '.',
43+
'Period': '.',
44+
'period': '.',
45+
'Quote': '\'',
46+
'quote': '\'',
47+
'double quote': '"',
48+
'Double quote': '"',
49+
};
50+
51+
export class TerminalVoiceSession extends Disposable {
52+
private _input: string = '';
53+
private _ghostText: IDecoration | undefined;
54+
private _decoration: IDecoration | undefined;
55+
private _marker: IMarker | undefined;
56+
private _ghostTextMarker: IMarker | undefined;
57+
private static _instance: TerminalVoiceSession | undefined = undefined;
58+
private _acceptTranscriptionScheduler: RunOnceScheduler | undefined;
59+
static getInstance(instantiationService: IInstantiationService): TerminalVoiceSession {
60+
if (!TerminalVoiceSession._instance) {
61+
TerminalVoiceSession._instance = instantiationService.createInstance(TerminalVoiceSession);
62+
}
63+
64+
return TerminalVoiceSession._instance;
65+
}
66+
private _cancellationTokenSource: CancellationTokenSource | undefined;
67+
private readonly _disposables: DisposableStore;
68+
constructor(
69+
@ISpeechService private readonly _speechService: ISpeechService,
70+
@ITerminalService private readonly _terminalService: ITerminalService,
71+
@IConfigurationService private readonly _configurationService: IConfigurationService,
72+
) {
73+
super();
74+
this._register(this._terminalService.onDidChangeActiveInstance(() => this.stop()));
75+
this._register(this._terminalService.onDidDisposeInstance(() => this.stop()));
76+
this._disposables = this._register(new DisposableStore());
77+
}
78+
79+
async start(): Promise<void> {
80+
this.stop();
81+
let voiceTimeout = this._configurationService.getValue<number>(AccessibilityVoiceSettingId.SpeechTimeout);
82+
if (!isNumber(voiceTimeout) || voiceTimeout < 0) {
83+
voiceTimeout = SpeechTimeoutDefault;
84+
}
85+
this._acceptTranscriptionScheduler = this._disposables.add(new RunOnceScheduler(() => {
86+
this._sendText();
87+
this.stop();
88+
}, voiceTimeout));
89+
this._cancellationTokenSource = new CancellationTokenSource();
90+
this._register(toDisposable(() => this._cancellationTokenSource?.dispose(true)));
91+
const session = await this._speechService.createSpeechToTextSession(this._cancellationTokenSource?.token, 'terminal');
92+
93+
this._disposables.add(session.onDidChange((e) => {
94+
if (this._cancellationTokenSource?.token.isCancellationRequested) {
95+
return;
96+
}
97+
switch (e.status) {
98+
case SpeechToTextStatus.Started:
99+
if (!this._decoration) {
100+
this._createDecoration();
101+
}
102+
break;
103+
case SpeechToTextStatus.Recognizing: {
104+
this._updateInput(e);
105+
this._renderGhostText(e);
106+
if (voiceTimeout > 0) {
107+
this._acceptTranscriptionScheduler!.cancel();
108+
}
109+
break;
110+
}
111+
case SpeechToTextStatus.Recognized:
112+
this._updateInput(e);
113+
if (voiceTimeout > 0) {
114+
this._acceptTranscriptionScheduler!.schedule();
115+
}
116+
break;
117+
case SpeechToTextStatus.Stopped:
118+
this.stop();
119+
break;
120+
}
121+
}));
122+
}
123+
stop(send?: boolean): void {
124+
this._setInactive();
125+
if (send) {
126+
this._acceptTranscriptionScheduler!.cancel();
127+
this._sendText();
128+
}
129+
this._ghostText = undefined;
130+
this._decoration = undefined;
131+
this._marker = undefined;
132+
this._ghostTextMarker = undefined;
133+
this._cancellationTokenSource?.cancel();
134+
this._disposables.clear();
135+
this._input = '';
136+
}
137+
138+
private _sendText(): void {
139+
this._terminalService.activeInstance?.sendText(this._input, false);
140+
alert(localize('terminalVoiceTextInserted', '{0} inserted', this._input));
141+
}
142+
143+
private _updateInput(e: ISpeechToTextEvent): void {
144+
if (e.text) {
145+
let input = e.text.replaceAll(/[.,?;!]/g, '');
146+
for (const symbol of Object.entries(symbolMap)) {
147+
input = input.replace(new RegExp('\\b' + symbol[0] + '\\b'), symbol[1]);
148+
}
149+
this._input = ' ' + input;
150+
}
151+
}
152+
153+
private _createDecoration(): void {
154+
const activeInstance = this._terminalService.activeInstance;
155+
const xterm = activeInstance?.xterm?.raw;
156+
if (!xterm) {
157+
return;
158+
}
159+
const onFirstLine = xterm.buffer.active.cursorY === 0;
160+
this._marker = activeInstance.registerMarker(onFirstLine ? 0 : -1);
161+
if (!this._marker) {
162+
return;
163+
}
164+
this._disposables.add(this._marker);
165+
this._decoration = xterm.registerDecoration({
166+
marker: this._marker,
167+
layer: 'top',
168+
x: xterm.buffer.active.cursorX ?? 0,
169+
});
170+
if (this._decoration) {
171+
this._disposables.add(this._decoration);
172+
}
173+
this._decoration?.onRender((e: HTMLElement) => {
174+
e.classList.add(...ThemeIcon.asClassNameArray(Codicon.micFilled), 'terminal-voice', 'recording');
175+
e.style.transform = onFirstLine ? 'translate(10px, -2px)' : 'translate(-6px, -5px)';
176+
});
177+
}
178+
179+
private _setInactive(): void {
180+
this._decoration?.element?.classList.remove('recording');
181+
}
182+
183+
private _renderGhostText(e: ISpeechToTextEvent): void {
184+
this._ghostText?.dispose();
185+
const text = e.text;
186+
if (!text) {
187+
return;
188+
}
189+
const activeInstance = this._terminalService.activeInstance;
190+
const xterm = activeInstance?.xterm?.raw;
191+
if (!xterm) {
192+
return;
193+
}
194+
this._ghostTextMarker = activeInstance.registerMarker();
195+
if (!this._ghostTextMarker) {
196+
return;
197+
}
198+
this._disposables.add(this._ghostTextMarker);
199+
const onFirstLine = xterm.buffer.active.cursorY === 0;
200+
this._ghostText = xterm.registerDecoration({
201+
marker: this._ghostTextMarker,
202+
layer: 'top',
203+
x: onFirstLine ? xterm.buffer.active.cursorX + 4 : xterm.buffer.active.cursorX + 1,
204+
});
205+
if (this._ghostText) {
206+
this._disposables.add(this._ghostText);
207+
}
208+
this._ghostText?.onRender((e: HTMLElement) => {
209+
e.classList.add('terminal-voice-progress-text');
210+
e.textContent = text;
211+
e.style.width = (xterm.cols - xterm.buffer.active.cursorX) / xterm.cols * 100 + '%';
212+
});
213+
}
214+
}

‎src/vs/workbench/contrib/terminalContrib/voice/browser/terminalVoiceActions.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { localize2 } from '../../../../../nls.js';
7+
import { ContextKeyExpr } from '../../../../../platform/contextkey/common/contextkey.js';
8+
import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
9+
import { HasSpeechProvider } from '../../../speech/common/speechService.js';
10+
import { registerActiveInstanceAction, sharedWhenClause } from '../../../terminal/browser/terminalActions.js';
11+
import { TerminalCommandId } from '../../../terminal/common/terminal.js';
12+
import { TerminalVoiceSession } from './terminalVoice.js';
13+
14+
export function registerTerminalVoiceActions() {
15+
registerActiveInstanceAction({
16+
id: TerminalCommandId.StartVoice,
17+
title: localize2('workbench.action.terminal.startDictation', "Start Dictation in Terminal"),
18+
precondition: ContextKeyExpr.and(HasSpeechProvider, sharedWhenClause.terminalAvailable),
19+
f1: true,
20+
run: (activeInstance, c, accessor) => {
21+
const instantiationService = accessor.get(IInstantiationService);
22+
TerminalVoiceSession.getInstance(instantiationService).start();
23+
}
24+
});
25+
26+
registerActiveInstanceAction({
27+
id: TerminalCommandId.StopVoice,
28+
title: localize2('workbench.action.terminal.stopDictation', "Stop Dictation in Terminal"),
29+
precondition: ContextKeyExpr.and(HasSpeechProvider, sharedWhenClause.terminalAvailable),
30+
f1: true,
31+
run: (activeInstance, c, accessor) => {
32+
const instantiationService = accessor.get(IInstantiationService);
33+
TerminalVoiceSession.getInstance(instantiationService).stop(true);
34+
}
35+
});
36+
}

0 commit comments

Comments
 (0)