From aa7047b3e39bc6a1c95174d2cbbe0f97d6ba4260 Mon Sep 17 00:00:00 2001 From: Sharif Date: Thu, 11 May 2023 09:30:47 -0700 Subject: [PATCH 1/4] ACS captions --- Project/src/App.css | 4 ++ Project/src/MakeCall/CallCaption.js | 68 +++++++++++++++++++++++++++++ Project/src/MakeCall/CallCard.js | 29 +++++++++++- 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 Project/src/MakeCall/CallCaption.js diff --git a/Project/src/App.css b/Project/src/App.css index 70f40f48..e5f57ce7 100644 --- a/Project/src/App.css +++ b/Project/src/App.css @@ -1173,6 +1173,10 @@ h3 { width: 100%; max-width: 150px; } +.caption-area { + max-height: 300px; + overflow: auto; +} .custom-video-effects-buttons:not(.outgoing) { position: absolute; diff --git a/Project/src/MakeCall/CallCaption.js b/Project/src/MakeCall/CallCaption.js new file mode 100644 index 00000000..b7812ec1 --- /dev/null +++ b/Project/src/MakeCall/CallCaption.js @@ -0,0 +1,68 @@ +import React, { useState, useEffect } from "react"; +import { Features, ResultType } from '@azure/communication-calling'; + +// CallCaption react function component +const CallCaption = ({ call }) => { + // caption history state + const [captionHistory, setCaptionHistory] = useState([]); + let captions; + + useEffect(() => { + captions = call.feature(Features.Captions); + startCaptions(captions); + + return () => { + // cleanup + captions.off('isCaptionsActiveChanged', isCaptionsActiveHandler); + captions.off('captionsReceived', captionHandler); + }; + }, []); + + const startCaptions = async () => { + try { + if (!captions.isCaptionsActive) { + await captions.startCaptions({ spokenLanguage: 'en-us' }); + } + captions.on('isCaptionsActiveChanged', isCaptionsActiveHandler); + captions.on('captionsReceived', captionHandler); + } catch (e) { + console.error('startCaptions failed', e); + } + }; + + const isCaptionsActiveHandler = () => { + console.log('isCaptionsActiveChanged: ', captions.isCaptionsActive); + } + + const captionHandler = (captionData) => { + let mri = ''; + if (captionData.speaker.identifier.kind === 'communicationUser') { + mri = captionData.speaker.identifier.communicationUserId; + } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') { + mri = captionData.speaker.identifier.microsoftTeamsUserId; + } else if (captionData.speaker.identifier.kind === 'phoneNumber') { + mri = captionData.speaker.identifier.phoneNumber; + } + + const captionText = `${captionData.timestamp.toUTCString()} + ${captionData.speaker.displayName}: ${captionData.text}`; + + console.log(mri, captionText); + if (captionData.resultType === ResultType.Final) { + setCaptionHistory(oldCaptions => [...oldCaptions, captionText]); + } + + }; + + return ( +
+ { + captionHistory.map((caption, index) => ( +
{caption}
+ )) + } +
+ ); +}; + +export default CallCaption; \ No newline at end of file diff --git a/Project/src/MakeCall/CallCard.js b/Project/src/MakeCall/CallCard.js index d09931f2..6e394073 100644 --- a/Project/src/MakeCall/CallCard.js +++ b/Project/src/MakeCall/CallCard.js @@ -17,6 +17,7 @@ import { Label } from '@fluentui/react/lib/Label'; import { AzureLogger } from '@azure/logger'; import VolumeVisualizer from "./VolumeVisualizer"; import CurrentCallInformation from "./CurrentCallInformation"; +import CallCaption from "./CallCaption"; export default class CallCard extends React.Component { constructor(props) { @@ -52,6 +53,7 @@ export default class CallCard extends React.Component { showLocalVideo: false, callMessage: undefined, dominantSpeakerMode: false, + captionOn: false, dominantRemoteParticipant: undefined, logMediaStats: false, sentResolution: '', @@ -847,7 +849,7 @@ export default class CallCard extends React.Component { } this.toggleParticipantsCard()}> { @@ -986,6 +988,31 @@ export default class CallCard extends React.Component { } +
+ + Caption{' '} + + + +
+ } + styles={{ + text : { color: '#edebe9' }, + label: { color: '#edebe9' }, + }} + inlineLabel + onText="On" + offText="Off" + defaultChecked={this.state.captionOn} + onChange={() => { this.setState({ captionOn: !this.state.captionOn })}} + /> + + { + this.state.captionOn && + + } + } From 806a744c093876c611c121e4ef667d50995c58be Mon Sep 17 00:00:00 2001 From: Chukwuebuka Nwankwo Date: Thu, 11 May 2023 16:55:21 -0500 Subject: [PATCH 2/4] Communication AI ui panel --- Project/src/MakeCall/CallCard.js | 29 ++++- .../CommunicationAI/CommunicationAI.js | 111 ++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 Project/src/MakeCall/CommunicationAI/CommunicationAI.js diff --git a/Project/src/MakeCall/CallCard.js b/Project/src/MakeCall/CallCard.js index 6e394073..73df5212 100644 --- a/Project/src/MakeCall/CallCard.js +++ b/Project/src/MakeCall/CallCard.js @@ -18,6 +18,7 @@ import { AzureLogger } from '@azure/logger'; import VolumeVisualizer from "./VolumeVisualizer"; import CurrentCallInformation from "./CurrentCallInformation"; import CallCaption from "./CallCaption"; +import CommunicationAI from "./CommunicationAI/CommunicationAI" export default class CallCard extends React.Component { constructor(props) { @@ -54,6 +55,7 @@ export default class CallCard extends React.Component { callMessage: undefined, dominantSpeakerMode: false, captionOn: false, + communicationAI: false, dominantRemoteParticipant: undefined, logMediaStats: false, sentResolution: '', @@ -1012,7 +1014,32 @@ export default class CallCard extends React.Component { this.state.captionOn && } - + +
+ + Communication AI{' '} + + + +
+ } + styles={{ + text : { color: '#edebe9' }, + label: { color: '#edebe9' }, + }} + inlineLabel + onText="On" + offText="Off" + defaultChecked={this.state.communicationAI} + onChange={() => { this.setState({ communicationAI: !this.state.communicationAI })}} + /> + + { + this.state.communicationAI && + + } + } diff --git a/Project/src/MakeCall/CommunicationAI/CommunicationAI.js b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js new file mode 100644 index 00000000..b854561f --- /dev/null +++ b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js @@ -0,0 +1,111 @@ +import React, { useState, useEffect } from "react"; +import { Features, ResultType } from '@azure/communication-calling'; +import { Dropdown } from '@fluentui/react/lib/Dropdown'; + + +const CommunicationAI = ({ call }) => { + const [captionsStarted, setCaptionsStarted] = useState(false) + const [captionHistory, setCaptionHistory] = useState([]); + const [lastSummary, setlastSummary] = useState(""); + const [lastfeedBack, setlastfeedBack] = useState(""); + const [promptResponse, setPromptResponse] = useState("") + const options = [ + { key: 'getSummary', text: 'Get Summary'}, + { key: 'getPersonalFeedBack', text: 'Get Personal Feedback' }, + ] + + let captions; + + useEffect(() => { + captions = call.feature(Features.Captions); + startCaptions(captions); + + return () => { + // cleanup + captions.off('captionsReceived', captionHandler); + }; + }, []); + + const startCaptions = async () => { + try { + if (!captions.isCaptionsActive || !captionsStarted) { + await captions.startCaptions({ spokenLanguage: 'en-us' }); + setCaptionsStarted(!captionsStarted); + } + captions.on('captionsReceived', captionHandler); + } catch (e) { + console.error('startCaptions failed', e); + } + }; + + const captionHandler = (captionData) => { + let mri = ''; + if (captionData.speaker.identifier.kind === 'communicationUser') { + mri = captionData.speaker.identifier.communicationUserId; + } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') { + mri = captionData.speaker.identifier.microsoftTeamsUserId; + } else if (captionData.speaker.identifier.kind === 'phoneNumber') { + mri = captionData.speaker.identifier.phoneNumber; + } + + const captionText = `${captionData.speaker.displayName}: ${captionData.text}`; + + console.log(mri, captionText); + if (captionData.resultType === ResultType.Final) { + setCaptionHistory(oldCaptions => [...oldCaptions, captionText]); + } + + }; + + + const getSummary = () => { + // placeholder until we get server response + setPromptResponse("FHL <=> Get summary") + } + + const getPersonalFeedback = () => { + // placeholder until we get server response + setPromptResponse("FHL <=> Get Personal FeedBack") + + } + + const onChangeHandler = (e, item) => { + let communicationAiOption = item.key; + switch (communicationAiOption) { + case "getSummary": + getSummary() + break + case "getPersonalFeedBack": + getPersonalFeedback() + break + } + + } + + return ( + <> +
+ +
+
+

{promptResponse}

+

{"Place holder of captions data (will be removed)"}

+
+ { + captionHistory.map((caption, index) => ( +
{caption}
+ )) + } +
+
+ + ); +}; + +export default CommunicationAI; \ No newline at end of file From d7493c66e96b068a6914e1cbe57289d9241f34e2 Mon Sep 17 00:00:00 2001 From: Chukwuebuka Nwankwo Date: Fri, 12 May 2023 09:58:12 -0500 Subject: [PATCH 3/4] Implementation to get server response --- .../CommunicationAI/CommunicationAI.js | 63 ++++++++++--------- Project/src/Utils/Utils.js | 26 ++++++++ 2 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Project/src/MakeCall/CommunicationAI/CommunicationAI.js b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js index b854561f..dc210cb5 100644 --- a/Project/src/MakeCall/CommunicationAI/CommunicationAI.js +++ b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js @@ -1,21 +1,24 @@ import React, { useState, useEffect } from "react"; import { Features, ResultType } from '@azure/communication-calling'; import { Dropdown } from '@fluentui/react/lib/Dropdown'; +import { utils, acsOpenAiPromptsApi } from "../../Utils/Utils"; const CommunicationAI = ({ call }) => { const [captionsStarted, setCaptionsStarted] = useState(false) const [captionHistory, setCaptionHistory] = useState([]); const [lastSummary, setlastSummary] = useState(""); - const [lastfeedBack, setlastfeedBack] = useState(""); + const [captionsSummaryIndex, setCaptionsSummaryIndex] = useState(0); + const [lastFeedBack, setLastFeedBack] = useState(""); + const [captionsFeedbackIndex, setCaptionsFeedbackIndex] = useState(0); const [promptResponse, setPromptResponse] = useState("") + const [dropDownLabel, setDropDownLabel] = useState("") const options = [ { key: 'getSummary', text: 'Get Summary'}, { key: 'getPersonalFeedBack', text: 'Get Personal Feedback' }, ] - + let displayName = "Testusera"; let captions; - useEffect(() => { captions = call.feature(Features.Captions); startCaptions(captions); @@ -58,19 +61,27 @@ const CommunicationAI = ({ call }) => { }; - const getSummary = () => { - // placeholder until we get server response - setPromptResponse("FHL <=> Get summary") + const getSummary = async () => { + const currentCaptionsData = captionHistory.slice(captionsSummaryIndex); + let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.summary, displayName, lastSummary, currentCaptionsData); + const content = response.choices[0].message.content; + setlastSummary(content) + setCaptionsSummaryIndex(captionHistory.length); + setPromptResponse(content) } - const getPersonalFeedback = () => { - // placeholder until we get server response - setPromptResponse("FHL <=> Get Personal FeedBack") - + const getPersonalFeedback = async () => { + const currentCaptionsData = captionHistory.slice(captionsFeedbackIndex); + let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.feedback, displayName, lastFeedBack, currentCaptionsData) + const content = response.choices[0].message.content; + setLastFeedBack(content) + setCaptionsFeedbackIndex(captionHistory.length); + setPromptResponse(content) } const onChangeHandler = (e, item) => { let communicationAiOption = item.key; + setDropDownLabel(item.text); switch (communicationAiOption) { case "getSummary": getSummary() @@ -84,26 +95,18 @@ const CommunicationAI = ({ call }) => { return ( <> -
- -
-
-

{promptResponse}

-

{"Place holder of captions data (will be removed)"}

-
- { - captionHistory.map((caption, index) => ( -
{caption}
- )) - } -
-
+
+ +
+
+

{promptResponse}

+
); }; diff --git a/Project/src/Utils/Utils.js b/Project/src/Utils/Utils.js index 6913cd05..de2400d7 100644 --- a/Project/src/Utils/Utils.js +++ b/Project/src/Utils/Utils.js @@ -6,6 +6,12 @@ import { } from '@azure/communication-common'; import axios from 'axios'; +export const acsOpenAiPromptsApi = { + base: 'https://acsopenaigateway.azurewebsites.net/api/', + summary: 'getSummary', + feedback: 'getPersonalFeedback' +} + export const utils = { getAppServiceUrl: () => { return window.location.origin; @@ -51,6 +57,26 @@ export const utils = { } throw new Error('Failed to get ACS User Acccess token for the given OneSignal Registration Token'); }, + sendCaptionsDataToAcsOpenAI: async (apiEndpoint, participantName, lastResponse, newCaptionsData) => { + let response = await axios({ + url: acsOpenAiPromptsApi.base + apiEndpoint, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': "*", + 'x-functions-key': 'PUT_FUNCTION_KEY' + }, + data: { + "CurrentParticipant": participantName, + "Captions": lastResponse, + "LastSummary": newCaptionsData + + } + }); + if (response.status === 200) { + return response.data; + } + }, getIdentifierText: (identifier) => { if (isCommunicationUserIdentifier(identifier)) { return identifier.communicationUserId; From 19aaaf17236491263c47a0d9714ff3b21226337f Mon Sep 17 00:00:00 2001 From: Chukwuebuka Nwankwo Date: Fri, 12 May 2023 10:01:10 -0500 Subject: [PATCH 4/4] Fixed request body --- Project/src/Utils/Utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project/src/Utils/Utils.js b/Project/src/Utils/Utils.js index de2400d7..e7c0f341 100644 --- a/Project/src/Utils/Utils.js +++ b/Project/src/Utils/Utils.js @@ -68,8 +68,8 @@ export const utils = { }, data: { "CurrentParticipant": participantName, - "Captions": lastResponse, - "LastSummary": newCaptionsData + "Captions": newCaptionsData, + "LastSummary": lastResponse } });