cnwankwo/ai panel

Project/src/App.css

@@ -1173,6 +1173,10 @@ h3 {
   width: 100%;
   max-width: 150px;
 }
+.caption-area {
+  max-height: 300px;
+  overflow: auto;
+}
 
 .custom-video-effects-buttons:not(.outgoing) {
   position: absolute;

Project/src/MakeCall/CallCaption.js

@@ -0,0 +1,68 @@
+import React, { useState, useEffect } from "react";
+import { Features, ResultType } from '@azure/communication-calling';
+
+// CallCaption react function component
+const CallCaption = ({ call }) => {
+    // caption history state
+    const [captionHistory, setCaptionHistory] = useState([]);
+    let captions;
+
+    useEffect(() => {
+        captions = call.feature(Features.Captions);
+        startCaptions(captions);
+
+        return () => {
+            // cleanup
+            captions.off('isCaptionsActiveChanged', isCaptionsActiveHandler);
+            captions.off('captionsReceived', captionHandler);
+        };
+    }, []);
+
+    const startCaptions = async () => {
+        try {
+            if (!captions.isCaptionsActive) {
+                await captions.startCaptions({ spokenLanguage: 'en-us' });
+            }
+            captions.on('isCaptionsActiveChanged', isCaptionsActiveHandler);
+            captions.on('captionsReceived', captionHandler);
+        } catch (e) {
+            console.error('startCaptions failed', e);
+        }
+    };
+
+    const isCaptionsActiveHandler = () => {
+        console.log('isCaptionsActiveChanged: ', captions.isCaptionsActive);
+    }
+
+    const captionHandler = (captionData) => {
+        let mri = '';
+        if (captionData.speaker.identifier.kind === 'communicationUser') {
+            mri = captionData.speaker.identifier.communicationUserId;
+        } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') {
+            mri = captionData.speaker.identifier.microsoftTeamsUserId;
+        } else if (captionData.speaker.identifier.kind === 'phoneNumber') {
+            mri = captionData.speaker.identifier.phoneNumber;
+        }
+
+        const captionText = `${captionData.timestamp.toUTCString()}
+                ${captionData.speaker.displayName}: ${captionData.text}`;
+
+        console.log(mri, captionText);
+        if (captionData.resultType === ResultType.Final) {
+            setCaptionHistory(oldCaptions => [...oldCaptions, captionText]);
+        }
+
+    };
+
+    return (
+        <div id="captionArea" className="caption-area">
+            {
+                captionHistory.map((caption, index) => (
+                    <div key={index}>{caption}</div>
+                ))
+            }
+        </div>
+    );
+};
+
+export default CallCaption;

Project/src/MakeCall/CallCard.js

@@ -17,6 +17,8 @@ import { Label } from '@fluentui/react/lib/Label';
 import { AzureLogger } from '@azure/logger';
 import VolumeVisualizer from "./VolumeVisualizer";
 import CurrentCallInformation from "./CurrentCallInformation";
+import CallCaption from "./CallCaption";
+import CommunicationAI from "./CommunicationAI/CommunicationAI"
 
 export default class CallCard extends React.Component {
     constructor(props) {
@@ -52,6 +54,8 @@ export default class CallCard extends React.Component {
             showLocalVideo: false,
             callMessage: undefined,
             dominantSpeakerMode: false,
+            captionOn: false,
+            communicationAI: false,
             dominantRemoteParticipant: undefined,
             logMediaStats: false,
             sentResolution: '',
@@ -847,7 +851,7 @@ export default class CallCard extends React.Component {
                                     }
                                 </span>
                                 <span className="in-call-button"
-                                    title={`${!this.state.showParticipantsCard ? `Show Participants` : `Hide Participants`}`}
+                                    title={`${!this.state.showParticipantsCard ? `Show Participants and Caption Panel` : `Hide Participants and Caption Panel`}`}
                                     variant="secondary"
                                     onClick={() => this.toggleParticipantsCard()}>
                                     {
@@ -986,6 +990,56 @@ export default class CallCard extends React.Component {
                                 }
                             </ul>
                         </div>
+                        <div className="participants-panel mt-1 mb-3">
+                                <Toggle label={
+                                        <div>
+                                            Caption{' '}
+                                            <TooltipHost content={`Turn on Captions to see the conversation script`}>
+                                                <Icon iconName="Info" aria-label="Info tooltip" />
+                                            </TooltipHost>
+                                        </div>
+                                    }
+                                    styles={{
+                                        text : { color: '#edebe9' },
+                                        label: { color: '#edebe9' },
+                                    }}
+                                    inlineLabel
+                                    onText="On"
+                                    offText="Off"
+                                    defaultChecked={this.state.captionOn}
+                                    onChange={() => { this.setState({ captionOn: !this.state.captionOn })}}
+                                />
+
+                                {
+                                    this.state.captionOn &&
+                                    <CallCaption call={this.call} />
+                                }
+                        </div>
+                        <div className="participants-panel mt-1 mb-3">
+                                <Toggle label={
+                                        <div>
+                                            Communication AI{' '}
+                                            <TooltipHost content={`Turn on Communication AI`}>
+                                                <Icon iconName="Info" aria-label="Info tooltip" />
+                                            </TooltipHost>
+                                        </div>
+                                    }
+                                    styles={{
+                                        text : { color: '#edebe9' },
+                                        label: { color: '#edebe9' },
+                                    }}
+                                    inlineLabel
+                                    onText="On"
+                                    offText="Off"
+                                    defaultChecked={this.state.communicationAI}
+                                    onChange={() => { this.setState({ communicationAI: !this.state.communicationAI })}}
+                                />
+
+                                {
+                                    this.state.communicationAI &&
+                                    <CommunicationAI call={this.call} />
+                                }
+                        </div>
                     </div>
                 }
             </div>

Project/src/MakeCall/CommunicationAI/CommunicationAI.js

@@ -0,0 +1,114 @@
+import React, { useState, useEffect } from "react";
+import { Features, ResultType } from '@azure/communication-calling';
+import { Dropdown } from '@fluentui/react/lib/Dropdown';
+import { utils, acsOpenAiPromptsApi } from "../../Utils/Utils";
+
+
+const CommunicationAI = ({ call }) => {
+    const [captionsStarted, setCaptionsStarted] = useState(false)
+    const [captionHistory, setCaptionHistory] = useState([]);
+    const [lastSummary, setlastSummary] = useState("");
+    const [captionsSummaryIndex, setCaptionsSummaryIndex] = useState(0);
+    const [lastFeedBack, setLastFeedBack] = useState("");
+    const [captionsFeedbackIndex, setCaptionsFeedbackIndex] = useState(0);
+    const [promptResponse, setPromptResponse] = useState("")
+    const [dropDownLabel, setDropDownLabel] = useState("")
+    const options = [
+        { key: 'getSummary', text: 'Get Summary'},
+        { key: 'getPersonalFeedBack', text: 'Get Personal Feedback' },
+    ]
+    let displayName = "Testusera";
+    let captions;
+    useEffect(() => {
+        captions = call.feature(Features.Captions);
+        startCaptions(captions);
+
+        return () => {
+            // cleanup
+            captions.off('captionsReceived', captionHandler);
+        };
+    }, []);
+
+    const startCaptions = async () => {
+        try {
+            if (!captions.isCaptionsActive || !captionsStarted) {
+                await captions.startCaptions({ spokenLanguage: 'en-us' });
+                setCaptionsStarted(!captionsStarted);
+            }
+            captions.on('captionsReceived', captionHandler);
+        } catch (e) {
+            console.error('startCaptions failed', e);
+        }
+    };
+
+    const captionHandler = (captionData) => {
+        let mri = '';
+        if (captionData.speaker.identifier.kind === 'communicationUser') {
+            mri = captionData.speaker.identifier.communicationUserId;
+        } else if (captionData.speaker.identifier.kind === 'microsoftTeamsUser') {
+            mri = captionData.speaker.identifier.microsoftTeamsUserId;
+        } else if (captionData.speaker.identifier.kind === 'phoneNumber') {
+            mri = captionData.speaker.identifier.phoneNumber;
+        }
+
+        const captionText = `${captionData.speaker.displayName}: ${captionData.text}`;
+
+        console.log(mri, captionText);
+        if (captionData.resultType === ResultType.Final) {
+            setCaptionHistory(oldCaptions => [...oldCaptions, captionText]);
+        }
+
+    };
+
+
+    const getSummary = async () => {    
+        const currentCaptionsData =  captionHistory.slice(captionsSummaryIndex);
+        let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.summary, displayName, lastSummary, currentCaptionsData);
+        const content = response.choices[0].message.content;
+        setlastSummary(content)
+        setCaptionsSummaryIndex(captionHistory.length);
+        setPromptResponse(content)
+    }
+
+    const getPersonalFeedback = async () => {
+        const currentCaptionsData =  captionHistory.slice(captionsFeedbackIndex);
+        let response = await utils.sendCaptionsDataToAcsOpenAI(acsOpenAiPromptsApi.feedback, displayName, lastFeedBack, currentCaptionsData)
+        const content = response.choices[0].message.content;
+        setLastFeedBack(content)
+        setCaptionsFeedbackIndex(captionHistory.length);
+        setPromptResponse(content)
+    }
+
+    const onChangeHandler = (e, item) => {
+        let communicationAiOption = item.key;
+        setDropDownLabel(item.text);
+        switch (communicationAiOption) {
+            case "getSummary":
+                getSummary()
+                break
+            case "getPersonalFeedBack":
+                getPersonalFeedback()
+                break
+        }
+
+    }
+
+    return (
+        <>
+            <div id="" className="">
+                <Dropdown
+                    placeholder="Select an option"
+                    label= {dropDownLabel}
+                    options={options}
+                    styles={{dropdown: { width: 300 },}}
+                    onChange={onChangeHandler}
+                />
+            </div>
+            <div id="communicationResponse" className="">           
+                <p>{promptResponse}</p>
+            </div>
+        </>
+    );
+};
+
+export default CommunicationAI;

Project/src/Utils/Utils.js

@@ -6,6 +6,12 @@ import {
 } from '@azure/communication-common';
 import axios from 'axios';
 
+export const acsOpenAiPromptsApi = {
+    base: 'https://acsopenaigateway.azurewebsites.net/api/',
+    summary: 'getSummary',
+    feedback: 'getPersonalFeedback'
+}
+
 export const utils = {
     getAppServiceUrl: () => {
         return window.location.origin;
@@ -51,6 +57,26 @@ export const utils = {
         }
         throw new Error('Failed to get ACS User Acccess token for the given OneSignal Registration Token');
     },
+    sendCaptionsDataToAcsOpenAI: async (apiEndpoint, participantName, lastResponse, newCaptionsData) => {
+        let response = await axios({
+            url: acsOpenAiPromptsApi.base + apiEndpoint,
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Access-Control-Allow-Origin': "*",
+                'x-functions-key': 'PUT_FUNCTION_KEY'
+            },
+            data: {
+                "CurrentParticipant": participantName,
+                "Captions": newCaptionsData,
+                "LastSummary": lastResponse
+
+            }
+        });
+        if (response.status === 200) {
+            return response.data;
+        }
+    },
     getIdentifierText: (identifier) => {
         if (isCommunicationUserIdentifier(identifier)) {
             return identifier.communicationUserId;