Skip to content

Commit d2fa22d

Browse files
meganroggemeganrogge
committed
1 parent 141aa85 commit d2fa22d

File tree

2 files changed

+233
-1
lines changed

2 files changed

+233
-1
lines changed

‎src/vs/workbench/contrib/terminal/browser/terminalActions.ts

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import { ICommandService } from '../../../../platform/commands/common/commands.j
2929
import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js';
3030
import { ContextKeyExpr } from '../../../../platform/contextkey/common/contextkey.js';
3131
import { FileKind } from '../../../../platform/files/common/files.js';
32-
import { ServicesAccessor } from '../../../../platform/instantiation/common/instantiation.js';
32+
import { IInstantiationService, ServicesAccessor } from '../../../../platform/instantiation/common/instantiation.js';
3333
import { KeybindingWeight } from '../../../../platform/keybinding/common/keybindingsRegistry.js';
3434
import { ILabelService } from '../../../../platform/label/common/label.js';
3535
import { IListService } from '../../../../platform/list/browser/listService.js';
@@ -52,6 +52,7 @@ import { IWorkbenchEnvironmentService } from '../../../services/environment/comm
5252
import { IPreferencesService } from '../../../services/preferences/common/preferences.js';
5353
import { IRemoteAgentService } from '../../../services/remote/common/remoteAgentService.js';
5454
import { accessibleViewCurrentProviderId, accessibleViewIsShown, accessibleViewOnLastLine } from '../../accessibility/browser/accessibilityConfiguration.js';
55+
import { HasSpeechProvider } from '../../speech/common/speechService.js';
5556
import { IRemoteTerminalAttachTarget, ITerminalProfileResolverService, ITerminalProfileService, TERMINAL_VIEW_ID, TerminalCommandId } from '../common/terminal.js';
5657
import { TerminalContextKeys } from '../common/terminalContextKey.js';
5758
import { terminalStrings } from '../common/terminalStrings.js';
@@ -61,6 +62,7 @@ import { getColorClass, getIconId, getUriClasses } from './terminalIcon.js';
6162
import { killTerminalIcon, newTerminalIcon } from './terminalIcons.js';
6263
import { ITerminalQuickPickItem } from './terminalProfileQuickpick.js';
6364
import { TerminalTabList } from './terminalTabsList.js';
65+
import { TerminalVoiceSession } from './terminalVoice.js';
6466

6567
export const switchTerminalActionViewItemSeparator = '\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500';
6668
export const switchTerminalShowTabsTitle = localize('showTerminalTabs', "Show Tabs");
@@ -1393,6 +1395,28 @@ export function registerTerminalActions() {
13931395
}
13941396
}
13951397
});
1398+
1399+
registerActiveInstanceAction({
1400+
id: TerminalCommandId.StartVoice,
1401+
title: localize2('workbench.action.terminal.startDictation', "Start Dictation in Terminal"),
1402+
precondition: ContextKeyExpr.and(HasSpeechProvider, sharedWhenClause.terminalAvailable),
1403+
f1: true,
1404+
run: (activeInstance, c, accessor) => {
1405+
const instantiationService = accessor.get(IInstantiationService);
1406+
TerminalVoiceSession.getInstance(instantiationService).start();
1407+
}
1408+
});
1409+
1410+
registerActiveInstanceAction({
1411+
id: TerminalCommandId.StopVoice,
1412+
title: localize2('workbench.action.terminal.stopDictation', "Stop Dictation in Terminal"),
1413+
precondition: ContextKeyExpr.and(HasSpeechProvider, sharedWhenClause.terminalAvailable),
1414+
f1: true,
1415+
run: (activeInstance, c, accessor) => {
1416+
const instantiationService = accessor.get(IInstantiationService);
1417+
TerminalVoiceSession.getInstance(instantiationService).stop(true);
1418+
}
1419+
});
13961420
}
13971421

13981422
interface IRemoteTerminalPick extends IQuickPickItem {

‎src/vs/workbench/contrib/terminal/browser/terminalVoice.ts

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { RunOnceScheduler } from '../../../../base/common/async.js';
7+
import { CancellationTokenSource } from '../../../../base/common/cancellation.js';
8+
import { Codicon } from '../../../../base/common/codicons.js';
9+
import { Disposable, DisposableStore, toDisposable } from '../../../../base/common/lifecycle.js';
10+
import { ThemeIcon } from '../../../../base/common/themables.js';
11+
import { isNumber } from '../../../../base/common/types.js';
12+
import { localize } from '../../../../nls.js';
13+
import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js';
14+
import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
15+
import { SpeechTimeoutDefault } from '../../accessibility/browser/accessibilityConfiguration.js';
16+
import { ISpeechService, AccessibilityVoiceSettingId, ISpeechToTextEvent, SpeechToTextStatus } from '../../speech/common/speechService.js';
17+
import { ITerminalService } from './terminal.js';
18+
import type { IMarker, IDecoration } from '@xterm/xterm';
19+
import { alert } from '../../../../base/browser/ui/aria/aria.js';
20+
21+
22+
const symbolMap: { [key: string]: string } = {
23+
'Ampersand': '&',
24+
'ampersand': '&',
25+
'Dollar': '$',
26+
'dollar': '$',
27+
'Percent': '%',
28+
'percent': '%',
29+
'Asterisk': '*',
30+
'asterisk': '*',
31+
'Plus': '+',
32+
'plus': '+',
33+
'Equals': '=',
34+
'equals': '=',
35+
'Exclamation': '!',
36+
'exclamation': '!',
37+
'Slash': '/',
38+
'slash': '/',
39+
'Backslash': '\\',
40+
'backslash': '\\',
41+
'Dot': '.',
42+
'dot': '.',
43+
'Period': '.',
44+
'period': '.',
45+
'Quote': '\'',
46+
'quote': '\'',
47+
'double quote': '"',
48+
'Double quote': '"',
49+
};
50+
51+
export class TerminalVoiceSession extends Disposable {
52+
private _input: string = '';
53+
private _ghostText: IDecoration | undefined;
54+
private _decoration: IDecoration | undefined;
55+
private _marker: IMarker | undefined;
56+
private _ghostTextMarker: IMarker | undefined;
57+
private static _instance: TerminalVoiceSession | undefined = undefined;
58+
private _acceptTranscriptionScheduler: RunOnceScheduler | undefined;
59+
static getInstance(instantiationService: IInstantiationService): TerminalVoiceSession {
60+
if (!TerminalVoiceSession._instance) {
61+
TerminalVoiceSession._instance = instantiationService.createInstance(TerminalVoiceSession);
62+
}
63+
64+
return TerminalVoiceSession._instance;
65+
}
66+
private _cancellationTokenSource: CancellationTokenSource | undefined;
67+
private readonly _disposables: DisposableStore;
68+
constructor(
69+
@ISpeechService private readonly _speechService: ISpeechService,
70+
@ITerminalService private readonly _terminalService: ITerminalService,
71+
@IConfigurationService private readonly _configurationService: IConfigurationService,
72+
) {
73+
super();
74+
this._register(this._terminalService.onDidChangeActiveInstance(() => this.stop()));
75+
this._register(this._terminalService.onDidDisposeInstance(() => this.stop()));
76+
this._disposables = this._register(new DisposableStore());
77+
}
78+
79+
async start(): Promise<void> {
80+
this.stop();
81+
let voiceTimeout = this._configurationService.getValue<number>(AccessibilityVoiceSettingId.SpeechTimeout);
82+
if (!isNumber(voiceTimeout) || voiceTimeout < 0) {
83+
voiceTimeout = SpeechTimeoutDefault;
84+
}
85+
this._acceptTranscriptionScheduler = this._disposables.add(new RunOnceScheduler(() => {
86+
this._sendText();
87+
this.stop();
88+
}, voiceTimeout));
89+
this._cancellationTokenSource = new CancellationTokenSource();
90+
this._register(toDisposable(() => this._cancellationTokenSource?.dispose(true)));
91+
const session = await this._speechService.createSpeechToTextSession(this._cancellationTokenSource?.token, 'terminal');
92+
93+
this._disposables.add(session.onDidChange((e) => {
94+
if (this._cancellationTokenSource?.token.isCancellationRequested) {
95+
return;
96+
}
97+
switch (e.status) {
98+
case SpeechToTextStatus.Started:
99+
if (!this._decoration) {
100+
this._createDecoration();
101+
}
102+
break;
103+
case SpeechToTextStatus.Recognizing: {
104+
this._updateInput(e);
105+
this._renderGhostText(e);
106+
if (voiceTimeout > 0) {
107+
this._acceptTranscriptionScheduler!.cancel();
108+
}
109+
break;
110+
}
111+
case SpeechToTextStatus.Recognized:
112+
this._updateInput(e);
113+
if (voiceTimeout > 0) {
114+
this._acceptTranscriptionScheduler!.schedule();
115+
}
116+
break;
117+
case SpeechToTextStatus.Stopped:
118+
this.stop();
119+
break;
120+
}
121+
}));
122+
}
123+
stop(send?: boolean): void {
124+
this._setInactive();
125+
if (send) {
126+
this._acceptTranscriptionScheduler!.cancel();
127+
this._sendText();
128+
}
129+
this._marker?.dispose();
130+
this._ghostTextMarker?.dispose();
131+
this._ghostText?.dispose();
132+
this._ghostText = undefined;
133+
this._decoration?.dispose();
134+
this._decoration = undefined;
135+
this._cancellationTokenSource?.cancel();
136+
this._disposables.clear();
137+
this._input = '';
138+
}
139+
140+
private _sendText(): void {
141+
this._terminalService.activeInstance?.sendText(this._input, false);
142+
alert(localize('terminalVoiceTextInserted', '{0} inserted', this._input));
143+
}
144+
145+
private _updateInput(e: ISpeechToTextEvent): void {
146+
if (e.text) {
147+
let input = e.text.replaceAll(/[.,?;!]/g, '');
148+
for (const symbol of Object.entries(symbolMap)) {
149+
input = input.replace(new RegExp('\\b' + symbol[0] + '\\b'), symbol[1]);
150+
}
151+
this._input = ' ' + input;
152+
}
153+
}
154+
155+
private _createDecoration(): void {
156+
const activeInstance = this._terminalService.activeInstance;
157+
const xterm = activeInstance?.xterm?.raw;
158+
if (!xterm) {
159+
return;
160+
}
161+
const onFirstLine = xterm.buffer.active.cursorY === 0;
162+
this._marker = activeInstance.registerMarker(onFirstLine ? 0 : -1);
163+
if (!this._marker) {
164+
return;
165+
}
166+
this._decoration = xterm.registerDecoration({
167+
marker: this._marker,
168+
layer: 'top',
169+
x: xterm.buffer.active.cursorX ?? 0,
170+
});
171+
this._decoration?.onRender((e: HTMLElement) => {
172+
e.classList.add(...ThemeIcon.asClassNameArray(Codicon.micFilled), 'terminal-voice', 'recording');
173+
e.style.transform = onFirstLine ? 'translate(10px, -2px)' : 'translate(-6px, -5px)';
174+
});
175+
}
176+
177+
private _setInactive(): void {
178+
this._decoration?.element?.classList.remove('recording');
179+
}
180+
181+
private _renderGhostText(e: ISpeechToTextEvent): void {
182+
this._ghostText?.dispose();
183+
const text = e.text;
184+
if (!text) {
185+
return;
186+
}
187+
const activeInstance = this._terminalService.activeInstance;
188+
const xterm = activeInstance?.xterm?.raw;
189+
if (!xterm) {
190+
return;
191+
}
192+
this._ghostTextMarker = activeInstance.registerMarker();
193+
if (!this._ghostTextMarker) {
194+
return;
195+
}
196+
const onFirstLine = xterm.buffer.active.cursorY === 0;
197+
this._ghostText = xterm.registerDecoration({
198+
marker: this._ghostTextMarker,
199+
layer: 'top',
200+
x: onFirstLine ? xterm.buffer.active.cursorX + 4 : xterm.buffer.active.cursorX + 1,
201+
});
202+
this._ghostText?.onRender((e: HTMLElement) => {
203+
e.classList.add('terminal-voice-progress-text');
204+
e.textContent = text;
205+
e.style.width = (xterm.cols - xterm.buffer.active.cursorX) / xterm.cols * 100 + '%';
206+
});
207+
}
208+
}

0 commit comments

Comments
 (0)