diff --git a/Project/src/App.css b/Project/src/App.css
index 70f40f48..e5f57ce7 100644
--- a/Project/src/App.css
+++ b/Project/src/App.css
@@ -1173,6 +1173,10 @@ h3 {
width: 100%;
max-width: 150px;
}
+.caption-area {
+ max-height: 300px;
+ overflow: auto;
+}
.custom-video-effects-buttons:not(.outgoing) {
position: absolute;
diff --git a/Project/src/MakeCall/CallCaption.js b/Project/src/MakeCall/CallCaption.js
new file mode 100644
index 00000000..b7812ec1
--- /dev/null
+++ b/Project/src/MakeCall/CallCaption.js
@@ -0,0 +1,68 @@
+import React, { useState, useEffect } from "react";
+import { Features, ResultType } from '@azure/communication-calling';
+
+// CallCaption react function component
+const CallCaption = ({ call }) => {
+ // caption history state
+ const [captionHistory, setCaptionHistory] = useState([]);
+ let captions;
+
+ useEffect(() => {
+ captions = call.feature(Features.Captions);
+ startCaptions(captions);
+
+ return () => {
+ // cleanup
+ captions.off('isCaptionsActiveChanged', isCaptionsActiveHandler);
+ captions.off('captionsReceived', captionHandler);
+ };
+ }, []);
+
+ const startCaptions = async () => {
+ try {
+ if (!captions.isCaptionsActive) {
+ await captions.startCaptions({ spokenLanguage: 'en-us' });
+ }
+ captions.on('isCaptionsActiveChanged', isCaptionsActiveHandler);
+ captions.on('captionsReceived', captionHandler);
+ } catch (e) {
+ console.error('startCaptions failed', e);
+ }
+ };
+
+ const isCaptionsActiveHandler = () => {
+ console.log('isCaptionsActiveChanged: ', captions.isCaptionsActive);
+ }
+
+ const captionHandler = (captionData) => {
+ let mri = '';
+ if (captionData.speaker.identifier.kind === 'communicationUser') {
+ mri = captionData.speaker.identifier.communicationUserId;
+ } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') {
+ mri = captionData.speaker.identifier.microsoftTeamsUserId;
+ } else if (captionData.speaker.identifier.kind === 'phoneNumber') {
+ mri = captionData.speaker.identifier.phoneNumber;
+ }
+
+ const captionText = `${captionData.timestamp.toUTCString()}
+ ${captionData.speaker.displayName}: ${captionData.text}`;
+
+ console.log(mri, captionText);
+ if (captionData.resultType === ResultType.Final) {
+ setCaptionHistory(oldCaptions => [...oldCaptions, captionText]);
+ }
+
+ };
+
+ return (
+
+ {
+ captionHistory.map((caption, index) => (
+
{caption}
+ ))
+ }
+
+ );
+};
+
+export default CallCaption;
\ No newline at end of file
diff --git a/Project/src/MakeCall/CallCard.js b/Project/src/MakeCall/CallCard.js
index d09931f2..73df5212 100644
--- a/Project/src/MakeCall/CallCard.js
+++ b/Project/src/MakeCall/CallCard.js
@@ -17,6 +17,8 @@ import { Label } from '@fluentui/react/lib/Label';
import { AzureLogger } from '@azure/logger';
import VolumeVisualizer from "./VolumeVisualizer";
import CurrentCallInformation from "./CurrentCallInformation";
+import CallCaption from "./CallCaption";
+import CommunicationAI from "./CommunicationAI/CommunicationAI"
export default class CallCard extends React.Component {
constructor(props) {
@@ -52,6 +54,8 @@ export default class CallCard extends React.Component {
showLocalVideo: false,
callMessage: undefined,
dominantSpeakerMode: false,
+ captionOn: false,
+ communicationAI: false,
dominantRemoteParticipant: undefined,
logMediaStats: false,
sentResolution: '',
@@ -847,7 +851,7 @@ export default class CallCard extends React.Component {
}
this.toggleParticipantsCard()}>
{
@@ -986,6 +990,56 @@ export default class CallCard extends React.Component {
}
+
+
+ Caption{' '}
+
+
+
+
+ }
+ styles={{
+ text : { color: '#edebe9' },
+ label: { color: '#edebe9' },
+ }}
+ inlineLabel
+ onText="On"
+ offText="Off"
+ defaultChecked={this.state.captionOn}
+ onChange={() => { this.setState({ captionOn: !this.state.captionOn })}}
+ />
+
+ {
+ this.state.captionOn &&
+
+ }
+
+
+
+ Communication AI{' '}
+
+
+
+
+ }
+ styles={{
+ text : { color: '#edebe9' },
+ label: { color: '#edebe9' },
+ }}
+ inlineLabel
+ onText="On"
+ offText="Off"
+ defaultChecked={this.state.communicationAI}
+ onChange={() => { this.setState({ communicationAI: !this.state.communicationAI })}}
+ />
+
+ {
+ this.state.communicationAI &&
+
+ }
+
}
diff --git a/Project/src/MakeCall/CommunicationAI/CommunicationAI.js b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js
new file mode 100644
index 00000000..dc210cb5
--- /dev/null
+++ b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js
@@ -0,0 +1,114 @@
+import React, { useState, useEffect } from "react";
+import { Features, ResultType } from '@azure/communication-calling';
+import { Dropdown } from '@fluentui/react/lib/Dropdown';
+import { utils, acsOpenAiPromptsApi } from "../../Utils/Utils";
+
+
+const CommunicationAI = ({ call }) => {
+ const [captionsStarted, setCaptionsStarted] = useState(false)
+ const [captionHistory, setCaptionHistory] = useState([]);
+ const [lastSummary, setlastSummary] = useState("");
+ const [captionsSummaryIndex, setCaptionsSummaryIndex] = useState(0);
+ const [lastFeedBack, setLastFeedBack] = useState("");
+ const [captionsFeedbackIndex, setCaptionsFeedbackIndex] = useState(0);
+ const [promptResponse, setPromptResponse] = useState("")
+ const [dropDownLabel, setDropDownLabel] = useState("")
+ const options = [
+ { key: 'getSummary', text: 'Get Summary'},
+ { key: 'getPersonalFeedBack', text: 'Get Personal Feedback' },
+ ]
+ let displayName = "Testusera";
+ let captions;
+ useEffect(() => {
+ captions = call.feature(Features.Captions);
+ startCaptions(captions);
+
+ return () => {
+ // cleanup
+ captions.off('captionsReceived', captionHandler);
+ };
+ }, []);
+
+ const startCaptions = async () => {
+ try {
+ if (!captions.isCaptionsActive || !captionsStarted) {
+ await captions.startCaptions({ spokenLanguage: 'en-us' });
+ setCaptionsStarted(!captionsStarted);
+ }
+ captions.on('captionsReceived', captionHandler);
+ } catch (e) {
+ console.error('startCaptions failed', e);
+ }
+ };
+
+ const captionHandler = (captionData) => {
+ let mri = '';
+ if (captionData.speaker.identifier.kind === 'communicationUser') {
+ mri = captionData.speaker.identifier.communicationUserId;
+ } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') {
+ mri = captionData.speaker.identifier.microsoftTeamsUserId;
+ } else if (captionData.speaker.identifier.kind === 'phoneNumber') {
+ mri = captionData.speaker.identifier.phoneNumber;
+ }
+
+ const captionText = `${captionData.speaker.displayName}: ${captionData.text}`;
+
+ console.log(mri, captionText);
+ if (captionData.resultType === ResultType.Final) {
+ setCaptionHistory(oldCaptions => [...oldCaptions, captionText]);
+ }
+
+ };
+
+
+ const getSummary = async () => {
+ const currentCaptionsData = captionHistory.slice(captionsSummaryIndex);
+ let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.summary, displayName, lastSummary, currentCaptionsData);
+ const content = response.choices[0].message.content;
+ setlastSummary(content)
+ setCaptionsSummaryIndex(captionHistory.length);
+ setPromptResponse(content)
+ }
+
+ const getPersonalFeedback = async () => {
+ const currentCaptionsData = captionHistory.slice(captionsFeedbackIndex);
+ let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.feedback, displayName, lastFeedBack, currentCaptionsData)
+ const content = response.choices[0].message.content;
+ setLastFeedBack(content)
+ setCaptionsFeedbackIndex(captionHistory.length);
+ setPromptResponse(content)
+ }
+
+ const onChangeHandler = (e, item) => {
+ let communicationAiOption = item.key;
+ setDropDownLabel(item.text);
+ switch (communicationAiOption) {
+ case "getSummary":
+ getSummary()
+ break
+ case "getPersonalFeedBack":
+ getPersonalFeedback()
+ break
+ }
+
+ }
+
+ return (
+ <>
+
+
+
+
+ >
+ );
+};
+
+export default CommunicationAI;
\ No newline at end of file
diff --git a/Project/src/Utils/Utils.js b/Project/src/Utils/Utils.js
index 6913cd05..e7c0f341 100644
--- a/Project/src/Utils/Utils.js
+++ b/Project/src/Utils/Utils.js
@@ -6,6 +6,12 @@ import {
} from '@azure/communication-common';
import axios from 'axios';
+export const acsOpenAiPromptsApi = {
+ base: 'https://acsopenaigateway.azurewebsites.net/api/',
+ summary: 'getSummary',
+ feedback: 'getPersonalFeedback'
+}
+
export const utils = {
getAppServiceUrl: () => {
return window.location.origin;
@@ -51,6 +57,26 @@ export const utils = {
}
throw new Error('Failed to get ACS User Acccess token for the given OneSignal Registration Token');
},
+ sendCaptionsDataToAcsOpenAI: async (apiEndpoint, participantName, lastResponse, newCaptionsData) => {
+ let response = await axios({
+ url: acsOpenAiPromptsApi.base + apiEndpoint,
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'Access-Control-Allow-Origin': "*",
+ 'x-functions-key': 'PUT_FUNCTION_KEY'
+ },
+ data: {
+ "CurrentParticipant": participantName,
+ "Captions": newCaptionsData,
+ "LastSummary": lastResponse
+
+ }
+ });
+ if (response.status === 200) {
+ return response.data;
+ }
+ },
getIdentifierText: (identifier) => {
if (isCommunicationUserIdentifier(identifier)) {
return identifier.communicationUserId;