Skip to content

Commit 6ad6b55

Browse files
hapticdatahapticdata
committed
Merge branch 'feature/settings'
2 parents fab92bf + 7cc756d commit 6ad6b55

File tree

10 files changed

+526
-45
lines changed

10 files changed

+526
-45
lines changed

‎src/App.scss‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
--Green-500: #0d9c53;
4444
--Green-700: #025022;
4545

46+
--Blue-400: #80c1ff;
4647
--Blue-500: #1f94ff;
4748
--Blue-800: #0f3557;
4849

@@ -111,10 +112,11 @@ body {
111112
flex-direction: column;
112113
}
113114

114-
@media (prefers-reduced-motion: no-preference) {}
115+
@media (prefers-reduced-motion: no-preference) {
116+
}
115117

116118
.streaming-console {
117-
background: var(--Neutral-5);
119+
background: var(--Neutral-15);
118120
color: var(--gray-300);
119121
display: flex;
120122
height: 100vh;
@@ -129,7 +131,7 @@ body {
129131
.disabled {
130132
pointer-events: none;
131133

132-
>* {
134+
> * {
133135
pointer-events: none;
134136
}
135137
}

‎src/components/control-tray/ControlTray.tsx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,14 @@ import { useWebcam } from "../../hooks/use-webcam";
2424
import { AudioRecorder } from "../../lib/audio-recorder";
2525
import AudioPulse from "../audio-pulse/AudioPulse";
2626
import "./control-tray.scss";
27+
import SettingsDialog from "../settings-dialog/SettingsDialog";
2728

2829
export type ControlTrayProps = {
2930
videoRef: RefObject<HTMLVideoElement>;
3031
children?: ReactNode;
3132
supportsVideo: boolean;
3233
onVideoStreamChange?: (stream: MediaStream | null) => void;
34+
hideSettings?: boolean;
3335
};
3436

3537
type MediaStreamButtonProps = {
@@ -61,6 +63,7 @@ function ControlTray({
6163
children,
6264
onVideoStreamChange = () => {},
6365
supportsVideo,
66+
hideSettings,
6467
}: ControlTrayProps) {
6568
const videoStreams = [useWebcam(), useScreenCapture()];
6669
const [activeVideoStream, setActiveVideoStream] =
@@ -210,6 +213,7 @@ function ControlTray({
210213
</div>
211214
<span className="text-indicator">Streaming</span>
212215
</div>
216+
{hideSettings ? "" : <SettingsDialog />}
213217
</section>
214218
);
215219
}

‎src/components/settings-dialog/ResponseModalitySelector.tsx‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { useCallback, useEffect, useState } from "react";
2+
import Select from "react-select";
3+
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
4+
5+
const responseOptions = [
6+
{ value: "audio", label: "audio" },
7+
{ value: "text", label: "text" },
8+
];
9+
10+
export default function ResponseModalitySelector() {
11+
const { config, setConfig } = useLiveAPIContext();
12+
13+
const [selectedOption, setSelectedOption] = useState<{
14+
value: string;
15+
label: string;
16+
} | null>(responseOptions[0]);
17+
18+
const updateConfig = useCallback(
19+
(modality: "audio" | "text" | undefined) => {
20+
setConfig({
21+
...config,
22+
generationConfig: {
23+
...config.generationConfig,
24+
responseModalities: modality,
25+
},
26+
});
27+
},
28+
[config, setConfig]
29+
);
30+
31+
return (
32+
<div className="select-group">
33+
<label htmlFor="response-modality-selector">Response modality</label>
34+
<Select
35+
id="response-modality-selector"
36+
className="react-select"
37+
classNamePrefix="react-select"
38+
styles={{
39+
control: (baseStyles) => ({
40+
...baseStyles,
41+
background: "var(--Neutral-15)",
42+
color: "var(--Neutral-90)",
43+
minHeight: "33px",
44+
maxHeight: "33px",
45+
border: 0,
46+
}),
47+
option: (styles, { isFocused, isSelected }) => ({
48+
...styles,
49+
backgroundColor: isFocused
50+
? "var(--Neutral-30)"
51+
: isSelected
52+
? "var(--Neutral-20)"
53+
: undefined,
54+
}),
55+
}}
56+
defaultValue={selectedOption}
57+
options={responseOptions}
58+
onChange={(e) => {
59+
setSelectedOption(e);
60+
if (e && (e.value === "audio" || e.value === "text")) {
61+
updateConfig(e.value);
62+
}
63+
}}
64+
/>
65+
</div>
66+
);
67+
}

‎src/components/settings-dialog/SettingsDialog.tsx‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import {
2+
ChangeEvent,
3+
FormEventHandler,
4+
useCallback,
5+
useMemo,
6+
useState,
7+
} from "react";
8+
import "./settings-dialog.scss";
9+
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
10+
import { LiveConfig } from "../../multimodal-live-types";
11+
import {
12+
FunctionDeclaration,
13+
FunctionDeclarationsTool,
14+
Tool,
15+
} from "@google/generative-ai";
16+
import VoiceSelector from "./VoiceSelector";
17+
import ResponseModalitySelector from "./ResponseModalitySelector";
18+
19+
export default function SettingsDialog() {
20+
const [open, setOpen] = useState(false);
21+
const { config, setConfig, connected } = useLiveAPIContext();
22+
const functionDeclarations: FunctionDeclaration[] = useMemo(() => {
23+
if (!Array.isArray(config.tools)) {
24+
return [];
25+
}
26+
return (config.tools as Tool[])
27+
.filter((t: Tool): t is FunctionDeclarationsTool =>
28+
Array.isArray((t as any).functionDeclarations)
29+
)
30+
.map((t) => t.functionDeclarations)
31+
.filter((fc) => !!fc)
32+
.flat();
33+
}, [config]);
34+
35+
const systemInstruction = useMemo(() => {
36+
const s = config.systemInstruction?.parts.find((p) => p.text)?.text || "";
37+
38+
return s;
39+
}, [config]);
40+
41+
const updateConfig: FormEventHandler<HTMLTextAreaElement> = useCallback(
42+
(event: ChangeEvent<HTMLTextAreaElement>) => {
43+
const newConfig: LiveConfig = {
44+
...config,
45+
systemInstruction: {
46+
parts: [{ text: event.target.value }],
47+
},
48+
};
49+
setConfig(newConfig);
50+
},
51+
[config, setConfig]
52+
);
53+
54+
const updateFunctionDescription = useCallback(
55+
(editedFdName: string, newDescription: string) => {
56+
const newConfig: LiveConfig = {
57+
...config,
58+
tools:
59+
config.tools?.map((tool) => {
60+
const fdTool = tool as FunctionDeclarationsTool;
61+
if (!Array.isArray(fdTool.functionDeclarations)) {
62+
return tool;
63+
}
64+
return {
65+
...tool,
66+
functionDeclarations: fdTool.functionDeclarations.map((fd) =>
67+
fd.name === editedFdName
68+
? { ...fd, description: newDescription }
69+
: fd
70+
),
71+
};
72+
}) || [],
73+
};
74+
setConfig(newConfig);
75+
},
76+
[config, setConfig]
77+
);
78+
79+
return (
80+
<div className="settings-dialog">
81+
<button
82+
className="action-button material-symbols-outlined"
83+
onClick={() => setOpen(!open)}
84+
>
85+
settings
86+
</button>
87+
<dialog className="dialog" style={{ display: open ? "block" : "none" }}>
88+
<div className={`dialog-container ${connected ? "disabled" : ""}`}>
89+
{connected && (
90+
<div className="connected-indicator">
91+
<p>
92+
These settings can only be applied before connecting and will
93+
override other settings.
94+
</p>
95+
</div>
96+
)}
97+
<div className="mode-selectors">
98+
<ResponseModalitySelector />
99+
<VoiceSelector />
100+
</div>
101+
102+
<h3>System Instructions</h3>
103+
<textarea
104+
className="system"
105+
onChange={updateConfig}
106+
value={systemInstruction}
107+
/>
108+
<h4>Function declarations</h4>
109+
<div className="function-declarations">
110+
<div className="fd-rows">
111+
{functionDeclarations.map((fd, fdKey) => (
112+
<div className="fd-row" key={`function-${fdKey}`}>
113+
<span className="fd-row-name">{fd.name}</span>
114+
<span className="fd-row-args">
115+
{Object.keys(fd.parameters?.properties || {}).map(
116+
(item, k) => (
117+
<span key={k}>{item}</span>
118+
)
119+
)}
120+
</span>
121+
<input
122+
key={`fd-${fd.description}`}
123+
className="fd-row-description"
124+
type="text"
125+
defaultValue={fd.description}
126+
onBlur={(e) =>
127+
updateFunctionDescription(fd.name, e.target.value)
128+
}
129+
/>
130+
</div>
131+
))}
132+
</div>
133+
</div>
134+
</div>
135+
</dialog>
136+
</div>
137+
);
138+
}

‎src/components/settings-dialog/VoiceSelector.tsx‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import { useCallback, useEffect, useState } from "react";
2+
import Select from "react-select";
3+
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
4+
5+
const voiceOptions = [
6+
{ value: "Puck", label: "Puck" },
7+
{ value: "Charon", label: "Charon" },
8+
{ value: "Kore", label: "Kore" },
9+
{ value: "Fenrir", label: "Fenrir" },
10+
{ value: "Aoede", label: "Aoede" },
11+
];
12+
13+
export default function VoiceSelector() {
14+
const { config, setConfig } = useLiveAPIContext();
15+
16+
useEffect(() => {
17+
const voiceName =
18+
config.generationConfig?.speechConfig?.voiceConfig?.prebuiltVoiceConfig
19+
?.voiceName || "Atari02";
20+
const voiceOption = { value: voiceName, label: voiceName };
21+
setSelectedOption(voiceOption);
22+
}, [config]);
23+
24+
const [selectedOption, setSelectedOption] = useState<{
25+
value: string;
26+
label: string;
27+
} | null>(voiceOptions[5]);
28+
29+
const updateConfig = useCallback(
30+
(voiceName: string) => {
31+
setConfig({
32+
...config,
33+
generationConfig: {
34+
...config.generationConfig,
35+
speechConfig: {
36+
voiceConfig: {
37+
prebuiltVoiceConfig: {
38+
voiceName: voiceName,
39+
},
40+
},
41+
},
42+
},
43+
});
44+
},
45+
[config, setConfig]
46+
);
47+
48+
return (
49+
<div className="select-group">
50+
<label htmlFor="voice-selector">Voice</label>
51+
<Select
52+
id="voice-selector"
53+
className="react-select"
54+
classNamePrefix="react-select"
55+
styles={{
56+
control: (baseStyles) => ({
57+
...baseStyles,
58+
background: "var(--Neutral-15)",
59+
color: "var(--Neutral-90)",
60+
minHeight: "33px",
61+
maxHeight: "33px",
62+
border: 0,
63+
}),
64+
option: (styles, { isFocused, isSelected }) => ({
65+
...styles,
66+
backgroundColor: isFocused
67+
? "var(--Neutral-30)"
68+
: isSelected
69+
? "var(--Neutral-20)"
70+
: undefined,
71+
}),
72+
}}
73+
value={selectedOption}
74+
defaultValue={selectedOption}
75+
options={voiceOptions}
76+
onChange={(e) => {
77+
setSelectedOption(e);
78+
if (e) {
79+
updateConfig(e.value);
80+
}
81+
}}
82+
/>
83+
</div>
84+
);
85+
}

0 commit comments

Comments
 (0)