diff --git a/Project/src/App.css b/Project/src/App.css index 70f40f48..e5f57ce7 100644 --- a/Project/src/App.css +++ b/Project/src/App.css @@ -1173,6 +1173,10 @@ h3 { width: 100%; max-width: 150px; } +.caption-area { + max-height: 300px; + overflow: auto; +} .custom-video-effects-buttons:not(.outgoing) { position: absolute; diff --git a/Project/src/MakeCall/CallCaption.js b/Project/src/MakeCall/CallCaption.js new file mode 100644 index 00000000..b7812ec1 --- /dev/null +++ b/Project/src/MakeCall/CallCaption.js @@ -0,0 +1,68 @@ +import React, { useState, useEffect } from "react"; +import { Features, ResultType } from '@azure/communication-calling'; + +// CallCaption react function component +const CallCaption = ({ call }) => { + // caption history state + const [captionHistory, setCaptionHistory] = useState([]); + let captions; + + useEffect(() => { + captions = call.feature(Features.Captions); + startCaptions(captions); + + return () => { + // cleanup + captions.off('isCaptionsActiveChanged', isCaptionsActiveHandler); + captions.off('captionsReceived', captionHandler); + }; + }, []); + + const startCaptions = async () => { + try { + if (!captions.isCaptionsActive) { + await captions.startCaptions({ spokenLanguage: 'en-us' }); + } + captions.on('isCaptionsActiveChanged', isCaptionsActiveHandler); + captions.on('captionsReceived', captionHandler); + } catch (e) { + console.error('startCaptions failed', e); + } + }; + + const isCaptionsActiveHandler = () => { + console.log('isCaptionsActiveChanged: ', captions.isCaptionsActive); + } + + const captionHandler = (captionData) => { + let mri = ''; + if (captionData.speaker.identifier.kind === 'communicationUser') { + mri = captionData.speaker.identifier.communicationUserId; + } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') { + mri = captionData.speaker.identifier.microsoftTeamsUserId; + } else if (captionData.speaker.identifier.kind === 'phoneNumber') { + mri = captionData.speaker.identifier.phoneNumber; + } + + const captionText = `${captionData.timestamp.toUTCString()} + ${captionData.speaker.displayName}: ${captionData.text}`; + + console.log(mri, captionText); + if (captionData.resultType === ResultType.Final) { + setCaptionHistory(oldCaptions => [...oldCaptions, captionText]); + } + + }; + + return ( +
+ { + captionHistory.map((caption, index) => ( +
{caption}
+ )) + } +
+ ); +}; + +export default CallCaption; \ No newline at end of file diff --git a/Project/src/MakeCall/CallCard.js b/Project/src/MakeCall/CallCard.js index d09931f2..73df5212 100644 --- a/Project/src/MakeCall/CallCard.js +++ b/Project/src/MakeCall/CallCard.js @@ -17,6 +17,8 @@ import { Label } from '@fluentui/react/lib/Label'; import { AzureLogger } from '@azure/logger'; import VolumeVisualizer from "./VolumeVisualizer"; import CurrentCallInformation from "./CurrentCallInformation"; +import CallCaption from "./CallCaption"; +import CommunicationAI from "./CommunicationAI/CommunicationAI" export default class CallCard extends React.Component { constructor(props) { @@ -52,6 +54,8 @@ export default class CallCard extends React.Component { showLocalVideo: false, callMessage: undefined, dominantSpeakerMode: false, + captionOn: false, + communicationAI: false, dominantRemoteParticipant: undefined, logMediaStats: false, sentResolution: '', @@ -847,7 +851,7 @@ export default class CallCard extends React.Component { } this.toggleParticipantsCard()}> { @@ -986,6 +990,56 @@ export default class CallCard extends React.Component { } +
+ + Caption{' '} + + + +
+ } + styles={{ + text : { color: '#edebe9' }, + label: { color: '#edebe9' }, + }} + inlineLabel + onText="On" + offText="Off" + defaultChecked={this.state.captionOn} + onChange={() => { this.setState({ captionOn: !this.state.captionOn })}} + /> + + { + this.state.captionOn && + + } + +
+ + Communication AI{' '} + + + +
+ } + styles={{ + text : { color: '#edebe9' }, + label: { color: '#edebe9' }, + }} + inlineLabel + onText="On" + offText="Off" + defaultChecked={this.state.communicationAI} + onChange={() => { this.setState({ communicationAI: !this.state.communicationAI })}} + /> + + { + this.state.communicationAI && + + } + } diff --git a/Project/src/MakeCall/CommunicationAI/CommunicationAI.js b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js new file mode 100644 index 00000000..dc210cb5 --- /dev/null +++ b/Project/src/MakeCall/CommunicationAI/CommunicationAI.js @@ -0,0 +1,114 @@ +import React, { useState, useEffect } from "react"; +import { Features, ResultType } from '@azure/communication-calling'; +import { Dropdown } from '@fluentui/react/lib/Dropdown'; +import { utils, acsOpenAiPromptsApi } from "../../Utils/Utils"; + + +const CommunicationAI = ({ call }) => { + const [captionsStarted, setCaptionsStarted] = useState(false) + const [captionHistory, setCaptionHistory] = useState([]); + const [lastSummary, setlastSummary] = useState(""); + const [captionsSummaryIndex, setCaptionsSummaryIndex] = useState(0); + const [lastFeedBack, setLastFeedBack] = useState(""); + const [captionsFeedbackIndex, setCaptionsFeedbackIndex] = useState(0); + const [promptResponse, setPromptResponse] = useState("") + const [dropDownLabel, setDropDownLabel] = useState("") + const options = [ + { key: 'getSummary', text: 'Get Summary'}, + { key: 'getPersonalFeedBack', text: 'Get Personal Feedback' }, + ] + let displayName = "Testusera"; + let captions; + useEffect(() => { + captions = call.feature(Features.Captions); + startCaptions(captions); + + return () => { + // cleanup + captions.off('captionsReceived', captionHandler); + }; + }, []); + + const startCaptions = async () => { + try { + if (!captions.isCaptionsActive || !captionsStarted) { + await captions.startCaptions({ spokenLanguage: 'en-us' }); + setCaptionsStarted(!captionsStarted); + } + captions.on('captionsReceived', captionHandler); + } catch (e) { + console.error('startCaptions failed', e); + } + }; + + const captionHandler = (captionData) => { + let mri = ''; + if (captionData.speaker.identifier.kind === 'communicationUser') { + mri = captionData.speaker.identifier.communicationUserId; + } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') { + mri = captionData.speaker.identifier.microsoftTeamsUserId; + } else if (captionData.speaker.identifier.kind === 'phoneNumber') { + mri = captionData.speaker.identifier.phoneNumber; + } + + const captionText = `${captionData.speaker.displayName}: ${captionData.text}`; + + console.log(mri, captionText); + if (captionData.resultType === ResultType.Final) { + setCaptionHistory(oldCaptions => [...oldCaptions, captionText]); + } + + }; + + + const getSummary = async () => { + const currentCaptionsData = captionHistory.slice(captionsSummaryIndex); + let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.summary, displayName, lastSummary, currentCaptionsData); + const content = response.choices[0].message.content; + setlastSummary(content) + setCaptionsSummaryIndex(captionHistory.length); + setPromptResponse(content) + } + + const getPersonalFeedback = async () => { + const currentCaptionsData = captionHistory.slice(captionsFeedbackIndex); + let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.feedback, displayName, lastFeedBack, currentCaptionsData) + const content = response.choices[0].message.content; + setLastFeedBack(content) + setCaptionsFeedbackIndex(captionHistory.length); + setPromptResponse(content) + } + + const onChangeHandler = (e, item) => { + let communicationAiOption = item.key; + setDropDownLabel(item.text); + switch (communicationAiOption) { + case "getSummary": + getSummary() + break + case "getPersonalFeedBack": + getPersonalFeedback() + break + } + + } + + return ( + <> +
+ +
+
+

{promptResponse}

+
+ + ); +}; + +export default CommunicationAI; \ No newline at end of file diff --git a/Project/src/Utils/Utils.js b/Project/src/Utils/Utils.js index 6913cd05..e7c0f341 100644 --- a/Project/src/Utils/Utils.js +++ b/Project/src/Utils/Utils.js @@ -6,6 +6,12 @@ import { } from '@azure/communication-common'; import axios from 'axios'; +export const acsOpenAiPromptsApi = { + base: 'https://acsopenaigateway.azurewebsites.net/api/', + summary: 'getSummary', + feedback: 'getPersonalFeedback' +} + export const utils = { getAppServiceUrl: () => { return window.location.origin; @@ -51,6 +57,26 @@ export const utils = { } throw new Error('Failed to get ACS User Acccess token for the given OneSignal Registration Token'); }, + sendCaptionsDataToAcsOpenAI: async (apiEndpoint, participantName, lastResponse, newCaptionsData) => { + let response = await axios({ + url: acsOpenAiPromptsApi.base + apiEndpoint, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': "*", + 'x-functions-key': 'PUT_FUNCTION_KEY' + }, + data: { + "CurrentParticipant": participantName, + "Captions": newCaptionsData, + "LastSummary": lastResponse + + } + }); + if (response.status === 200) { + return response.data; + } + }, getIdentifierText: (identifier) => { if (isCommunicationUserIdentifier(identifier)) { return identifier.communicationUserId;