Skip to content

Commit 2c3d404

Browse files
trivikrtrivikr
authored
feat: turn note text to speech using Amazon Polly (#19)
1 parent a98c3bf commit 2c3d404

File tree

8 files changed

+175
-11
lines changed

8 files changed

+175
-11
lines changed

‎packages/frontend/README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@
33
- This package contains frontend code which performs:
44
- Put, Get, Delete operations on S3 client.
55
- StartStreamTranscription operation on TranscribeStreaming client.
6+
- SynthesizeSpeech operation on Polly client.
67
- This is a create-react-app which creates minimized bundle on running `build`, and debugs it on running `start`.
78

8-
<details><summary>Click to view screen recording</summary>
9+
<details><summary>Click to view screen recordings</summary>
910
<p>
1011

1112
[![Screen recording](https://img.youtube.com/vi/qBltinDalzU/0.jpg)](https://www.youtube.com/watch?v=qBltinDalzU)
1213

1314
[![Screen recording](https://img.youtube.com/vi/fF9zd0YJn6A/0.jpg)](https://www.youtube.com/watch?v=fF9zd0YJn6A)
1415

16+
[![Screen recording](https://img.youtube.com/vi/tNI05dyeyqY/0.jpg)](https://www.youtube.com/watch?v=tNI05dyeyqY)
17+
1518
</p>
1619
</details>
1720

‎packages/frontend/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
"description": "Frontend for notes app created using modular AWS SDK for JavaScript",
66
"dependencies": {
77
"@aws-sdk/client-cognito-identity": "3.4.1",
8+
"@aws-sdk/client-polly": "3.4.1",
89
"@aws-sdk/client-s3": "3.4.1",
910
"@aws-sdk/client-transcribe-streaming": "3.4.1",
1011
"@aws-sdk/credential-provider-cognito-identity": "3.4.1",
12+
"@aws-sdk/polly-request-presigner": "3.4.1",
1113
"@aws-sdk/s3-request-presigner": "3.4.1",
1214
"@aws-sdk/util-create-request": "3.4.1",
1315
"@aws-sdk/util-format-url": "3.4.1",

‎packages/frontend/src/content/CreateNote.tsx‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@ import { GATEWAY_URL, MAX_FILE_SIZE } from "../config.json";
55
import { putObject } from "../libs";
66
import { HomeButton, ButtonSpinner, PageContainer } from "../components";
77
import { RecordAudioButton } from "./RecordAudioButton";
8+
import { PlayAudioButton } from "./PlayAudioButton";
89

910
const CreateNote = (props: RouteComponentProps) => {
1011
const [isLoading, setIsLoading] = useState(false);
1112
const [errorMsg, setErrorMsg] = useState("");
1213
const [noteContent, setNoteContent] = useState("");
13-
const [isRecording, setIsRecording] = useState(false);
1414
const [file, setFile] = useState();
1515

16+
const [isRecording, setIsRecording] = useState(false);
17+
const [isPlaying, setIsPlaying] = useState(false);
18+
1619
const handleSubmit = async (event: FormEvent<HTMLFormElement>) => {
1720
event.preventDefault();
1821

@@ -41,9 +44,8 @@ const CreateNote = (props: RouteComponentProps) => {
4144
}
4245
};
4346

44-
const noteContentAdditionalProps = isRecording
45-
? { disabled: true, value: noteContent }
46-
: {};
47+
const noteContentAdditionalProps =
48+
isRecording || isPlaying ? { disabled: true, value: noteContent } : {};
4749

4850
return (
4951
<PageContainer header={<HomeButton />}>
@@ -63,11 +65,20 @@ const CreateNote = (props: RouteComponentProps) => {
6365
{...noteContentAdditionalProps}
6466
/>
6567
</Form.Group>
66-
<RecordAudioButton
67-
isRecording={isRecording}
68-
setIsRecording={setIsRecording}
69-
setNoteContent={setNoteContent}
70-
/>
68+
<Form.Group>
69+
<RecordAudioButton
70+
disabled={isPlaying}
71+
isRecording={isRecording}
72+
setIsRecording={setIsRecording}
73+
setNoteContent={setNoteContent}
74+
/>
75+
<PlayAudioButton
76+
disabled={isRecording}
77+
isPlaying={isPlaying}
78+
setIsPlaying={setIsPlaying}
79+
noteContent={noteContent}
80+
/>
81+
</Form.Group>
7182
<Form.Group controlId="file">
7283
<Form.Label>Attachment</Form.Label>
7384
<Form.Control

‎packages/frontend/src/content/PlayAudioButton.tsx‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import React, { useRef, useState } from "react";
2+
import { Button, Alert } from "react-bootstrap";
3+
import { PlayCircle, StopFill } from "react-bootstrap-icons";
4+
import { getSynthesizedSpeechUrl } from "../libs/getSynthesizedSpeechUrl";
5+
6+
const PlayAudioButton = (props: {
7+
disabled: boolean;
8+
isPlaying: boolean;
9+
setIsPlaying: Function;
10+
noteContent: string;
11+
}) => {
12+
const { disabled, isPlaying, setIsPlaying, noteContent } = props;
13+
const audioPlayer = useRef<HTMLAudioElement>(null);
14+
const [audioUrl, setAudioUrl] = useState("");
15+
const [errorMsg, setErrorMsg] = useState("");
16+
17+
const togglePlay = async () => {
18+
if (isPlaying) {
19+
setIsPlaying(false);
20+
audioPlayer.current?.pause();
21+
audioPlayer.current?.load();
22+
} else {
23+
setIsPlaying(true);
24+
try {
25+
const audioUrl = await getSynthesizedSpeechUrl(noteContent);
26+
setAudioUrl(audioUrl.toString());
27+
audioPlayer.current?.load();
28+
audioPlayer.current?.play();
29+
audioPlayer.current?.addEventListener("ended", () => {
30+
setIsPlaying(false);
31+
});
32+
} catch (error) {
33+
setIsPlaying(false);
34+
audioPlayer.current?.pause();
35+
audioPlayer.current?.load();
36+
console.log(error);
37+
setErrorMsg(`${error.toString()}`);
38+
}
39+
}
40+
};
41+
42+
return (
43+
<>
44+
{errorMsg && <Alert variant="danger">{errorMsg}</Alert>}
45+
<audio ref={audioPlayer} src={audioUrl}></audio>
46+
<Button
47+
className="mx-2"
48+
variant={isPlaying ? "primary" : "outline-secondary"}
49+
size="sm"
50+
onClick={togglePlay}
51+
disabled={disabled}
52+
>
53+
{isPlaying ? <StopFill /> : <PlayCircle />}
54+
</Button>
55+
</>
56+
);
57+
};
58+
59+
export { PlayAudioButton };

‎packages/frontend/src/content/RecordAudioButton.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ import { pcmEncode } from "../libs/audioUtils";
88
import { getStreamTranscriptionResponse } from "../libs/getStreamTranscriptionResponse";
99

1010
const RecordAudioButton = (props: {
11+
disabled: boolean;
1112
isRecording: boolean;
1213
setIsRecording: Function;
1314
setNoteContent: Function;
1415
}) => {
15-
const { isRecording, setIsRecording, setNoteContent } = props;
16+
const { disabled, isRecording, setIsRecording, setNoteContent } = props;
1617
const [micStream, setMicStream] = useState<MicrophoneStream | undefined>();
1718
const [errorMsg, setErrorMsg] = useState("");
1819

@@ -110,6 +111,7 @@ const RecordAudioButton = (props: {
110111
variant={isRecording ? "primary" : "outline-secondary"}
111112
size="sm"
112113
onClick={toggleTrascription}
114+
disabled={disabled}
113115
>
114116
{isRecording ? <MicFill /> : <MicMute />}
115117
</Button>

‎packages/frontend/src/libs/getSynthesizedSpeechUrl.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { PollyClient } from "@aws-sdk/client-polly";
2+
import { getSynthesizeSpeechUrl } from "@aws-sdk/polly-request-presigner";
3+
import { fromCognitoIdentityPool } from "@aws-sdk/credential-provider-cognito-identity";
4+
import { CognitoIdentityClient } from "@aws-sdk/client-cognito-identity";
5+
import { IDENTITY_POOL_ID, REGION } from "../config.json";
6+
7+
const getSynthesizedSpeechUrl = (textToSynthesize: string) => {
8+
const client = new PollyClient({
9+
region: REGION,
10+
credentials: fromCognitoIdentityPool({
11+
client: new CognitoIdentityClient({ region: REGION }),
12+
identityPoolId: IDENTITY_POOL_ID,
13+
}),
14+
});
15+
16+
return getSynthesizeSpeechUrl({
17+
client,
18+
params: {
19+
OutputFormat: "mp3",
20+
Text: textToSynthesize,
21+
VoiceId: "Aditi",
22+
},
23+
});
24+
};
25+
26+
export { getSynthesizedSpeechUrl };

‎packages/infra/cdk/aws-sdk-js-notes-app-stack.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@ export class AwsSdkJsNotesAppStack extends cdk.Stack {
114114
})
115115
);
116116

117+
// Add policy to enable Amazon Polly text-to-speech
118+
unauthenticated.addManagedPolicy(
119+
iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonPollyFullAccess")
120+
);
121+
117122
new cognito.CfnIdentityPoolRoleAttachment(this, "role-attachment", {
118123
identityPoolId: identityPool.ref,
119124
roles: {

‎yarn.lock‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,9 +755,11 @@ __metadata:
755755
resolution: "@aws-sdk-notes-app/frontend@workspace:packages/frontend"
756756
dependencies:
757757
"@aws-sdk/client-cognito-identity": 3.4.1
758+
"@aws-sdk/client-polly": 3.4.1
758759
"@aws-sdk/client-s3": 3.4.1
759760
"@aws-sdk/client-transcribe-streaming": 3.4.1
760761
"@aws-sdk/credential-provider-cognito-identity": 3.4.1
762+
"@aws-sdk/polly-request-presigner": 3.4.1
761763
"@aws-sdk/s3-request-presigner": 3.4.1
762764
"@aws-sdk/util-create-request": 3.4.1
763765
"@aws-sdk/util-format-url": 3.4.1
@@ -903,6 +905,45 @@ __metadata:
903905
languageName: node
904906
linkType: hard
905907

908+
"@aws-sdk/client-polly@npm:3.4.1":
909+
version: 3.4.1
910+
resolution: "@aws-sdk/client-polly@npm:3.4.1"
911+
dependencies:
912+
"@aws-crypto/sha256-browser": ^1.0.0
913+
"@aws-crypto/sha256-js": ^1.0.0
914+
"@aws-sdk/config-resolver": 3.4.1
915+
"@aws-sdk/credential-provider-node": 3.4.1
916+
"@aws-sdk/fetch-http-handler": 3.4.1
917+
"@aws-sdk/hash-node": 3.4.1
918+
"@aws-sdk/invalid-dependency": 3.4.1
919+
"@aws-sdk/middleware-content-length": 3.4.1
920+
"@aws-sdk/middleware-host-header": 3.4.1
921+
"@aws-sdk/middleware-logger": 3.4.1
922+
"@aws-sdk/middleware-retry": 3.4.1
923+
"@aws-sdk/middleware-serde": 3.4.1
924+
"@aws-sdk/middleware-signing": 3.4.1
925+
"@aws-sdk/middleware-stack": 3.4.1
926+
"@aws-sdk/middleware-user-agent": 3.4.1
927+
"@aws-sdk/node-config-provider": 3.4.1
928+
"@aws-sdk/node-http-handler": 3.4.1
929+
"@aws-sdk/protocol-http": 3.4.1
930+
"@aws-sdk/smithy-client": 3.4.1
931+
"@aws-sdk/types": 3.4.1
932+
"@aws-sdk/url-parser": 3.4.1
933+
"@aws-sdk/url-parser-native": 3.4.1
934+
"@aws-sdk/util-base64-browser": 3.4.1
935+
"@aws-sdk/util-base64-node": 3.4.1
936+
"@aws-sdk/util-body-length-browser": 3.4.1
937+
"@aws-sdk/util-body-length-node": 3.4.1
938+
"@aws-sdk/util-user-agent-browser": 3.4.1
939+
"@aws-sdk/util-user-agent-node": 3.4.1
940+
"@aws-sdk/util-utf8-browser": 3.4.1
941+
"@aws-sdk/util-utf8-node": 3.4.1
942+
tslib: ^2.0.0
943+
checksum: af8d56bf5898e3baeaccc01403cde06b30017fbc767dd971c2b73ae5d68b8c8c7989cea50b2e4d69c64e66a58a9d27fcd2e3e6e72770b8d92090f976f38f5928
944+
languageName: node
945+
linkType: hard
946+
906947
"@aws-sdk/client-s3@npm:3.4.1":
907948
version: 3.4.1
908949
resolution: "@aws-sdk/client-s3@npm:3.4.1"
@@ -1451,6 +1492,21 @@ __metadata:
14511492
languageName: node
14521493
linkType: hard
14531494

1495+
"@aws-sdk/polly-request-presigner@npm:3.4.1":
1496+
version: 3.4.1
1497+
resolution: "@aws-sdk/polly-request-presigner@npm:3.4.1"
1498+
dependencies:
1499+
"@aws-sdk/protocol-http": 3.4.1
1500+
"@aws-sdk/signature-v4": 3.4.1
1501+
"@aws-sdk/smithy-client": 3.4.1
1502+
"@aws-sdk/types": 3.4.1
1503+
"@aws-sdk/util-create-request": 3.4.1
1504+
"@aws-sdk/util-format-url": 3.4.1
1505+
tslib: ^1.8.0
1506+
checksum: dc93cc1d3e0916d3414acfd63581b2b8bd9d8671832c87aa6bcca8709b0106a39c2f014ab5ca63eae23d2f420958d92b3c6cea66fd5975288ce3297a11afb107
1507+
languageName: node
1508+
linkType: hard
1509+
14541510
"@aws-sdk/property-provider@npm:3.4.1":
14551511
version: 3.4.1
14561512
resolution: "@aws-sdk/property-provider@npm:3.4.1"

0 commit comments

Comments
 (0)