|
| 1 | +import logging |
| 2 | +import uuid |
| 3 | +import base64 |
| 4 | +from typing import Any, Awaitable, Callable, Dict, Iterable, List, Optional, Text |
| 5 | + |
| 6 | +from rasa.core.channels.channel import InputChannel, OutputChannel, UserMessage |
| 7 | +import rasa.shared.utils.io |
| 8 | +from sanic import Blueprint, response |
| 9 | +from sanic.request import Request |
| 10 | +from sanic.response import HTTPResponse |
| 11 | +from socketio import AsyncServer |
| 12 | + |
| 13 | +from urllib.request import urlopen, Request |
| 14 | +from urllib.parse import urlencode |
| 15 | +import json |
| 16 | + |
| 17 | +logger = logging.getLogger(__name__) |
| 18 | + |
| 19 | +print('Hello from SocketVoice') |
| 20 | + |
| 21 | +class SocketVoiceBlueprint(Blueprint): |
| 22 | + def __init__(self, sio: AsyncServer, socketio_path, *args, **kwargs): |
| 23 | + self.sio = sio |
| 24 | + self.socketio_path = socketio_path |
| 25 | + super().__init__(*args, **kwargs) |
| 26 | + |
| 27 | + def register(self, app, options) -> None: |
| 28 | + self.sio.attach(app, self.socketio_path) |
| 29 | + super().register(app, options) |
| 30 | + |
| 31 | + |
| 32 | +class SocketIOVoiceOutput(OutputChannel): |
| 33 | + @classmethod |
| 34 | + def name(cls) -> Text: |
| 35 | + return "socketiovoice" |
| 36 | + |
| 37 | + def __init__(self, sio: AsyncServer, bot_message_evt: Text, botium_speech_url: Text, botium_speech_apikey: Text, botium_speech_language: Text, botium_speech_voice: Text) -> None: |
| 38 | + self.sio = sio |
| 39 | + self.bot_message_evt = bot_message_evt |
| 40 | + self.botium_speech_url = botium_speech_url |
| 41 | + self.botium_speech_apikey = botium_speech_apikey |
| 42 | + self.botium_speech_language = botium_speech_language |
| 43 | + self.botium_speech_voice = botium_speech_voice |
| 44 | + |
| 45 | + async def _send_message(self, socket_id: Text, response: Any) -> None: |
| 46 | + """Sends a message to the recipient using the bot event.""" |
| 47 | + |
| 48 | + if response.get("text"): |
| 49 | + q = { |
| 50 | + 'text': response['text'] |
| 51 | + } |
| 52 | + if self.botium_speech_voice: |
| 53 | + q['voice'] = self.botium_speech_voice |
| 54 | + |
| 55 | + audioEndpoint = f"{self.botium_speech_url}/api/tts/{self.botium_speech_language}?{urlencode(q)}" |
| 56 | + audio = urlopen(audioEndpoint).read() |
| 57 | + logger.debug(f"_send_message- Calling Speech Endpoint: {audioEndpoint}") |
| 58 | + |
| 59 | + audioBase64 = base64.b64encode(audio).decode('ascii') |
| 60 | + audioUri = "data:audio/wav;base64," + audioBase64 |
| 61 | + response['link'] = audioUri |
| 62 | + |
| 63 | + await self.sio.emit(self.bot_message_evt, response, room=socket_id) |
| 64 | + |
| 65 | + async def send_text_message( |
| 66 | + self, recipient_id: Text, text: Text, **kwargs: Any |
| 67 | + ) -> None: |
| 68 | + """Send a message through this channel.""" |
| 69 | + |
| 70 | + for message_part in text.strip().split("\n\n"): |
| 71 | + await self._send_message(recipient_id, {"text": message_part}) |
| 72 | + |
| 73 | + async def send_image_url( |
| 74 | + self, recipient_id: Text, image: Text, **kwargs: Any |
| 75 | + ) -> None: |
| 76 | + """Sends an image to the output""" |
| 77 | + |
| 78 | + message = {"attachment": {"type": "image", "payload": {"src": image}}} |
| 79 | + await self._send_message(recipient_id, message) |
| 80 | + |
| 81 | + async def send_text_with_buttons( |
| 82 | + self, |
| 83 | + recipient_id: Text, |
| 84 | + text: Text, |
| 85 | + buttons: List[Dict[Text, Any]], |
| 86 | + **kwargs: Any, |
| 87 | + ) -> None: |
| 88 | + """Sends buttons to the output.""" |
| 89 | + |
| 90 | + # split text and create a message for each text fragment |
| 91 | + # the `or` makes sure there is at least one message we can attach the quick |
| 92 | + # replies to |
| 93 | + message_parts = text.strip().split("\n\n") or [text] |
| 94 | + messages = [{"text": message, "quick_replies": []} for message in message_parts] |
| 95 | + |
| 96 | + # attach all buttons to the last text fragment |
| 97 | + for button in buttons: |
| 98 | + messages[-1]["quick_replies"].append( |
| 99 | + { |
| 100 | + "content_type": "text", |
| 101 | + "title": button["title"], |
| 102 | + "payload": button["payload"], |
| 103 | + } |
| 104 | + ) |
| 105 | + |
| 106 | + for message in messages: |
| 107 | + await self._send_message(recipient_id, message) |
| 108 | + |
| 109 | + async def send_elements( |
| 110 | + self, recipient_id: Text, elements: Iterable[Dict[Text, Any]], **kwargs: Any |
| 111 | + ) -> None: |
| 112 | + """Sends elements to the output.""" |
| 113 | + |
| 114 | + for element in elements: |
| 115 | + message = { |
| 116 | + "attachment": { |
| 117 | + "type": "template", |
| 118 | + "payload": {"template_type": "generic", "elements": element}, |
| 119 | + } |
| 120 | + } |
| 121 | + |
| 122 | + await self._send_message(recipient_id, message) |
| 123 | + |
| 124 | + async def send_custom_json( |
| 125 | + self, recipient_id: Text, json_message: Dict[Text, Any], **kwargs: Any |
| 126 | + ) -> None: |
| 127 | + """Sends custom json to the output""" |
| 128 | + |
| 129 | + json_message.setdefault("room", recipient_id) |
| 130 | + |
| 131 | + await self.sio.emit(self.bot_message_evt, **json_message) |
| 132 | + |
| 133 | + async def send_attachment( |
| 134 | + self, recipient_id: Text, attachment: Dict[Text, Any], **kwargs: Any |
| 135 | + ) -> None: |
| 136 | + """Sends an attachment to the user.""" |
| 137 | + await self._send_message(recipient_id, {"attachment": attachment}) |
| 138 | + |
| 139 | + |
| 140 | +class SocketIOVoiceInput(InputChannel): |
| 141 | + """A socket.io input channel.""" |
| 142 | + |
| 143 | + @classmethod |
| 144 | + def name(cls) -> Text: |
| 145 | + return "socketiovoice" |
| 146 | + |
| 147 | + @classmethod |
| 148 | + def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> InputChannel: |
| 149 | + credentials = credentials or {} |
| 150 | + return cls( |
| 151 | + credentials.get("user_message_evt", "user_uttered"), |
| 152 | + credentials.get("bot_message_evt", "bot_uttered"), |
| 153 | + credentials.get("namespace"), |
| 154 | + credentials.get("session_persistence", False), |
| 155 | + credentials.get("socketio_path", "/socket.io"), |
| 156 | + credentials.get("botium_speech_url"), |
| 157 | + credentials.get("botium_speech_apikey"), |
| 158 | + credentials.get("botium_speech_language", "en"), |
| 159 | + credentials.get("botium_speech_voice"), |
| 160 | + ) |
| 161 | + |
| 162 | + def __init__( |
| 163 | + self, |
| 164 | + user_message_evt: Text = "user_uttered", |
| 165 | + bot_message_evt: Text = "bot_uttered", |
| 166 | + namespace: Optional[Text] = None, |
| 167 | + session_persistence: bool = False, |
| 168 | + socketio_path: Optional[Text] = "/socket.io", |
| 169 | + botium_speech_url: Text = None, |
| 170 | + botium_speech_apikey: Optional[Text] = None, |
| 171 | + botium_speech_language: Text = "en", |
| 172 | + botium_speech_voice: Optional[Text] = False, |
| 173 | + ): |
| 174 | + self.bot_message_evt = bot_message_evt |
| 175 | + self.session_persistence = session_persistence |
| 176 | + self.user_message_evt = user_message_evt |
| 177 | + self.namespace = namespace |
| 178 | + self.socketio_path = socketio_path |
| 179 | + self.botium_speech_url = botium_speech_url |
| 180 | + self.botium_speech_apikey = botium_speech_apikey |
| 181 | + self.botium_speech_language = botium_speech_language |
| 182 | + self.botium_speech_voice = botium_speech_voice |
| 183 | + self.sio = None |
| 184 | + |
| 185 | + def get_output_channel(self) -> Optional["OutputChannel"]: |
| 186 | + if self.sio is None: |
| 187 | + rasa.shared.utils.io.raise_warning( |
| 188 | + "SocketIO output channel cannot be recreated. " |
| 189 | + "This is expected behavior when using multiple Sanic " |
| 190 | + "workers or multiple Rasa Open Source instances. " |
| 191 | + "Please use a different channel for external events in these " |
| 192 | + "scenarios." |
| 193 | + ) |
| 194 | + return |
| 195 | + return SocketIOVoiceOutput(self.sio, self.bot_message_evt, self.botium_speech_url, self.botium_speech_apikey, self.botium_speech_language, self.botium_speech_voice) |
| 196 | + |
| 197 | + def blueprint( |
| 198 | + self, on_new_message: Callable[[UserMessage], Awaitable[Any]] |
| 199 | + ) -> Blueprint: |
| 200 | + # Workaround so that socketio works with requests from other origins. |
| 201 | + # https://github.com/miguelgrinberg/python-socketio/issues/205#issuecomment-493769183 |
| 202 | + sio = AsyncServer(async_mode="sanic", cors_allowed_origins=[]) |
| 203 | + socketio_webhook = SocketVoiceBlueprint( |
| 204 | + sio, self.socketio_path, "socketio_webhook", __name__ |
| 205 | + ) |
| 206 | + |
| 207 | + # make sio object static to use in get_output_channel |
| 208 | + self.sio = sio |
| 209 | + |
| 210 | + @socketio_webhook.route("/", methods=["GET"]) |
| 211 | + async def health(_: Request) -> HTTPResponse: |
| 212 | + return response.json({"status": "ok"}) |
| 213 | + |
| 214 | + @sio.on("connect", namespace=self.namespace) |
| 215 | + async def connect(sid: Text, _) -> None: |
| 216 | + logger.debug(f"User {sid} connected to socketIO endpoint.") |
| 217 | + |
| 218 | + @sio.on("disconnect", namespace=self.namespace) |
| 219 | + async def disconnect(sid: Text) -> None: |
| 220 | + logger.debug(f"User {sid} disconnected from socketIO endpoint.") |
| 221 | + |
| 222 | + @sio.on("session_request", namespace=self.namespace) |
| 223 | + async def session_request(sid: Text, data: Optional[Dict]): |
| 224 | + if data is None: |
| 225 | + data = {} |
| 226 | + if "session_id" not in data or data["session_id"] is None: |
| 227 | + data["session_id"] = uuid.uuid4().hex |
| 228 | + if self.session_persistence: |
| 229 | + sio.enter_room(sid, data["session_id"]) |
| 230 | + await sio.emit("session_confirm", data["session_id"], room=sid) |
| 231 | + logger.debug(f"User {sid} connected to socketIO endpoint.") |
| 232 | + |
| 233 | + @sio.on(self.user_message_evt, namespace=self.namespace) |
| 234 | + async def handle_message(sid: Text, data: Dict) -> Any: |
| 235 | + output_channel = SocketIOVoiceOutput(sio, self.bot_message_evt, self.botium_speech_url, self.botium_speech_apikey, self.botium_speech_language, self.botium_speech_voice) |
| 236 | + |
| 237 | + if self.session_persistence: |
| 238 | + if not data.get("session_id"): |
| 239 | + rasa.shared.utils.io.raise_warning( |
| 240 | + "A message without a valid session_id " |
| 241 | + "was received. This message will be " |
| 242 | + "ignored. Make sure to set a proper " |
| 243 | + "session id using the " |
| 244 | + "`session_request` socketIO event." |
| 245 | + ) |
| 246 | + return |
| 247 | + sender_id = data["session_id"] |
| 248 | + else: |
| 249 | + sender_id = sid |
| 250 | + |
| 251 | + if data['message'].startswith('data:'): |
| 252 | + header, encoded = data['message'].split(",", 1) |
| 253 | + |
| 254 | + audioData = base64.b64decode(encoded.encode('ascii')) |
| 255 | + |
| 256 | + convertEndpoint = f"{self.botium_speech_url}/api/convert/WAVTOMONOWAV" |
| 257 | + logger.debug(f"handle_message - Calling Convert Endpoint: {convertEndpoint}") |
| 258 | + res = urlopen(Request(url=convertEndpoint, data=audioData, method='POST', headers= { 'content-type': 'audio/wav' })) |
| 259 | + audioDataWav = res.read() |
| 260 | + |
| 261 | + #with open('decoded_image.wav', 'wb') as file_to_save: |
| 262 | + # file_to_save.write(audioData) |
| 263 | + |
| 264 | + audioEndpoint = f"{self.botium_speech_url}/api/stt/{self.botium_speech_language}" |
| 265 | + logger.debug(f"handle_message - Calling Speech Endpoint: {audioEndpoint}") |
| 266 | + res = urlopen(Request(url=audioEndpoint, data=audioDataWav, method='POST', headers= { 'content-type': 'audio/wav' })) |
| 267 | + resJson = json.loads(res.read().decode('utf-8')) |
| 268 | + logger.debug(f"handle_message - Calling Speech Endpoint: {audioEndpoint} => {resJson}") |
| 269 | + message = resJson["text"] |
| 270 | + |
| 271 | + await sio.emit(self.user_message_evt, {"text": message}, room=sid) |
| 272 | + else: |
| 273 | + message = data['message'] |
| 274 | + |
| 275 | + message = UserMessage( |
| 276 | + message, output_channel, sender_id, input_channel=self.name() |
| 277 | + ) |
| 278 | + await on_new_message(message) |
| 279 | + |
| 280 | + return socketio_webhook |
0 commit comments