feat: Interface with Qwen Omni speech to text model

Author: zhanweizhang7

apps/locales/en_US/LC_MESSAGES/django.po

@@ -8648,4 +8648,13 @@ msgid "Multiple dialects, supporting 23 dialects"
 msgstr ""
 
 msgid "This interface is used to recognize short audio files within 60 seconds. Supports Mandarin Chinese, English, Cantonese, Japanese, Vietnamese, Malay, Indonesian, Filipino, Thai, Portuguese, Turkish, Arabic, Hindi, French, German, and 23 Chinese dialects."
+msgstr ""
+
+msgid "CueWord"
+msgstr ""
+
+msgid "If not passed, the default value is What is this audio saying? Only answer the audio content"
+msgstr ""
+
+msgid "The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text."
 msgstr ""

apps/locales/zh_CN/LC_MESSAGES/django.po

@@ -8774,4 +8774,13 @@ msgid "Multiple dialects, supporting 23 dialects"
 msgstr "多种方言，支持 23 种方言"
 
 msgid "This interface is used to recognize short audio files within 60 seconds. Supports Mandarin Chinese, English, Cantonese, Japanese, Vietnamese, Malay, Indonesian, Filipino, Thai, Portuguese, Turkish, Arabic, Hindi, French, German, and 23 Chinese dialects."
-msgstr "本接口用于识别 60 秒之内的短音频文件。支持中文普通话、英语、粤语、日语、越南语、马来语、印度尼西亚语、菲律宾语、泰语、葡萄牙语、土耳其语、阿拉伯语、印地语、法语、德语及 23 种汉语方言。"
+msgstr "本接口用于识别 60 秒之内的短音频文件。支持中文普通话、英语、粤语、日语、越南语、马来语、印度尼西亚语、菲律宾语、泰语、葡萄牙语、土耳其语、阿拉伯语、印地语、法语、德语及 23 种汉语方言。"
+
+msgid "CueWord"
+msgstr "提示词"
+
+msgid "If not passed, the default value is What is this audio saying? Only answer the audio content"
+msgstr "如果未传递，默认值为 这段音频在说什么，只回答音频的内容"
+
+msgid "The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text."
+msgstr "Qwen-Omni 系列模型支持输入多种模态的数据，包括视频、音频、图片、文本，并输出音频与文本"

apps/locales/zh_Hant/LC_MESSAGES/django.po

@@ -8774,4 +8774,13 @@ msgid "Multiple dialects, supporting 23 dialects"
 msgstr "多種方言，支持 23 種方言"
 
 msgid "This interface is used to recognize short audio files within 60 seconds. Supports Mandarin Chinese, English, Cantonese, Japanese, Vietnamese, Malay, Indonesian, Filipino, Thai, Portuguese, Turkish, Arabic, Hindi, French, German, and 23 Chinese dialects."
-msgstr "本介面用於識別 60 秒之內的短音頻文件。支援中文普通話、英語、粵語、日語、越南語、馬來語、印度尼西亞語、菲律賓語、泰語、葡萄牙語、土耳其語、阿拉伯語、印地語、法語、德語及 23 種漢語方言。"
+msgstr "本介面用於識別 60 秒之內的短音頻文件。支援中文普通話、英語、粵語、日語、越南語、馬來語、印度尼西亞語、菲律賓語、泰語、葡萄牙語、土耳其語、阿拉伯語、印地語、法語、德語及 23 種漢語方言。"
+
+msgid "CueWord"
+msgstr "提示詞"
+
+msgid "If not passed, the default value is What is this audio saying? Only answer the audio content"
+msgstr "如果未傳遞，預設值為這段音訊在說什麼，只回答音訊的內容"
+
+msgid "The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text."
+msgstr "Qwen-Omni系列模型支持輸入多種模態的數據，包括視頻、音訊、圖片、文字，並輸出音訊與文字"

apps/models_provider/impl/aliyun_bai_lian_model_provider/aliyun_bai_lian_model_provider.py

@@ -15,6 +15,7 @@
     AliyunBaiLianEmbeddingCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.image import QwenVLModelCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.llm import BaiLianLLMModelCredential
+from models_provider.impl.aliyun_bai_lian_model_provider.credential.omi_stt import AliyunBaiLianOmiSTTModelCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.reranker import \
     AliyunBaiLianRerankerCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.stt import AliyunBaiLianSTTModelCredential
@@ -23,6 +24,7 @@
 from models_provider.impl.aliyun_bai_lian_model_provider.model.embedding import AliyunBaiLianEmbedding
 from models_provider.impl.aliyun_bai_lian_model_provider.model.image import QwenVLChatModel
 from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
+from models_provider.impl.aliyun_bai_lian_model_provider.model.omi_stt import AliyunBaiLianOmiSpeechToText
 from models_provider.impl.aliyun_bai_lian_model_provider.model.reranker import AliyunBaiLianReranker
 from models_provider.impl.aliyun_bai_lian_model_provider.model.stt import AliyunBaiLianSpeechToText
 from models_provider.impl.aliyun_bai_lian_model_provider.model.tti import QwenTextToImageModel
@@ -33,6 +35,7 @@
 aliyun_bai_lian_model_credential = AliyunBaiLianRerankerCredential()
 aliyun_bai_lian_tts_model_credential = AliyunBaiLianTTSModelCredential()
 aliyun_bai_lian_stt_model_credential = AliyunBaiLianSTTModelCredential()
+aliyun_bai_lian_omi_stt_model_credential = AliyunBaiLianOmiSTTModelCredential()
 aliyun_bai_lian_embedding_model_credential = AliyunBaiLianEmbeddingCredential()
 aliyun_bai_lian_llm_model_credential = BaiLianLLMModelCredential()
 qwenvl_model_credential = QwenVLModelCredential()
@@ -73,7 +76,10 @@
                    ModelInfo('qwen-plus', '', ModelTypeConst.LLM, aliyun_bai_lian_llm_model_credential,
                              BaiLianChatModel),
                    ModelInfo('qwen-max', '', ModelTypeConst.LLM, aliyun_bai_lian_llm_model_credential,
-                             BaiLianChatModel)
+                             BaiLianChatModel),
+                   ModelInfo('qwen-omni-turbo',
+                             _('The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text.'),
+                             ModelTypeConst.STT, aliyun_bai_lian_omi_stt_model_credential, AliyunBaiLianOmiSpeechToText),
                    ]
 
 module_info_vl_list = [

apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py

@@ -0,0 +1,72 @@
+# coding=utf-8
+import traceback
+from typing import Dict, Any
+
+from common import forms
+from common.exception.app_exception import AppApiException
+from common.forms import BaseForm, PasswordInputField, TooltipLabel
+from models_provider.base_model_provider import BaseModelCredential, ValidCode
+from django.utils.translation import gettext as _
+
+class AliyunBaiLianOmiSTTModelParams(BaseForm):
+    CueWord = forms.TextInputField(
+        TooltipLabel(_('CueWord'), _('If not passed, the default value is What is this audio saying? Only answer the audio content')),
+        required=True,
+        default_value='这段音频在说什么，只回答音频的内容',
+    )
+
+
+class AliyunBaiLianOmiSTTModelCredential(BaseForm, BaseModelCredential):
+    api_key = PasswordInputField("API key", required=True)
+
+    def is_valid(self,
+                 model_type: str,
+                 model_name: str,
+                 model_credential: Dict[str, Any],
+                 model_params: Dict[str, Any],
+                 provider,
+                 raise_exception: bool = False
+                 ) -> bool:
+
+        model_type_list = provider.get_model_type_list()
+        if not any(mt.get('value') == model_type for mt in model_type_list):
+            raise AppApiException(
+                ValidCode.valid_error.value,
+                _('{model_type} Model type is not supported').format(model_type=model_type)
+            )
+
+        required_keys = ['api_key']
+        for key in required_keys:
+            if key not in model_credential:
+                if raise_exception:
+                    raise AppApiException(
+                        ValidCode.valid_error.value,
+                        _('{key} is required').format(key=key)
+                    )
+                return False
+
+        try:
+            model = provider.get_model(model_type, model_name, model_credential)
+        except Exception as e:
+            traceback.print_exc()
+            if isinstance(e, AppApiException):
+                raise e
+            if raise_exception:
+                raise AppApiException(
+                ValidCode.valid_error.value,
+                _('Verification failed, please check whether the parameters are correct: {error}').format(error=str(e))
+                )
+            return False
+        return True
+
+    def encryption_dict(self, model: Dict[str, object]) -> Dict[str, object]:
+
+        return {
+            **model,
+            'api_key': super().encryption(model.get('api_key', ''))
+        }
+
+
+    def get_model_params_setting_form(self, model_name):
+
+        return AliyunBaiLianOmiSTTModelParams()

apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py

@@ -0,0 +1,87 @@
+import base64
+import os
+import traceback
+from typing import Dict
+
+from openai import OpenAI
+
+from common.utils.logger import maxkb_logger
+from models_provider.base_model_provider import MaxKBBaseModel
+from models_provider.impl.base_stt import BaseSpeechToText
+
+
+class AliyunBaiLianOmiSpeechToText(MaxKBBaseModel, BaseSpeechToText):
+    api_key: str
+    model: str
+    params: dict
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.api_key = kwargs.get('api_key')
+        self.model = kwargs.get('model')
+        self.params = kwargs.get('params')
+
+    @staticmethod
+    def is_cache_model():
+        return False
+
+    @staticmethod
+    def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
+        return AliyunBaiLianOmiSpeechToText(
+            model=model_name,
+            api_key=model_credential.get('api_key'),
+            params= model_kwargs,
+            **model_kwargs
+        )
+
+
+    def check_auth(self):
+        cwd = os.path.dirname(os.path.abspath(__file__))
+        with open(f'{cwd}/iat_mp3_16k.mp3', 'rb') as audio_file:
+            self.speech_to_text(audio_file)
+
+
+
+    def speech_to_text(self, audio_file):
+        try:
+            client = OpenAI(
+                # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx",
+                api_key=self.api_key,
+                base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+            )
+
+            base64_audio = base64.b64encode(audio_file.read()).decode("utf-8")
+
+            completion = client.chat.completions.create(
+                model="qwen-omni-turbo-0119",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "input_audio",
+                                "input_audio": {
+                                    "data": f"data:;base64,{base64_audio}",
+                                    "format": "mp3",
+                                },
+                            },
+                            {"type": "text", "text": self.params.get('CueWord')},
+                        ],
+                    },
+                ],
+                # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"]
+                modalities=["text"],
+                audio={"voice": "Cherry", "format": "mp3"},
+                # stream 必须设置为 True，否则会报错
+                stream=True,
+                stream_options={"include_usage": True},
+            )
+            result = []
+            for chunk in completion:
+                if chunk.choices and hasattr(chunk.choices[0].delta, 'audio'):
+                    transcript = chunk.choices[0].delta.audio.get('transcript')
+                    result.append(transcript)
+            return "".join(result)
+
+        except Exception as err:
+            maxkb_logger.error(f":Error: {str(err)}: {traceback.format_exc()}")

apps/models_provider/impl/tencent_model_provider/model/stt.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import os
+import traceback
 from typing import Dict
 
 from tencentcloud.asr.v20190614 import asr_client, models
@@ -9,6 +10,7 @@
 from tencentcloud.common.profile.client_profile import ClientProfile
 from tencentcloud.common.profile.http_profile import HttpProfile
 
+from common.utils.logger import maxkb_logger
 from models_provider.base_model_provider import MaxKBBaseModel
 from models_provider.impl.base_stt import BaseSpeechToText