feat: Qwen2.5-omni-7b full modal speech recognition

Author: zhanweizhang7

apps/models_provider/impl/aliyun_bai_lian_model_provider/aliyun_bai_lian_model_provider.py

@@ -15,7 +15,7 @@
     AliyunBaiLianEmbeddingCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.image import QwenVLModelCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.llm import BaiLianLLMModelCredential
-from models_provider.impl.aliyun_bai_lian_model_provider.credential.omi_stt import AliyunBaiLianOmiSTTModelCredential
+from models_provider.impl.aliyun_bai_lian_model_provider.credential.omni_stt import AliyunBaiLianOmiSTTModelCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.reranker import \
     AliyunBaiLianRerankerCredential
 from models_provider.impl.aliyun_bai_lian_model_provider.credential.stt import AliyunBaiLianSTTModelCredential
@@ -24,7 +24,7 @@
 from models_provider.impl.aliyun_bai_lian_model_provider.model.embedding import AliyunBaiLianEmbedding
 from models_provider.impl.aliyun_bai_lian_model_provider.model.image import QwenVLChatModel
 from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
-from models_provider.impl.aliyun_bai_lian_model_provider.model.omi_stt import AliyunBaiLianOmiSpeechToText
+from models_provider.impl.aliyun_bai_lian_model_provider.model.omni_stt import AliyunBaiLianOmiSpeechToText
 from models_provider.impl.aliyun_bai_lian_model_provider.model.reranker import AliyunBaiLianReranker
 from models_provider.impl.aliyun_bai_lian_model_provider.model.stt import AliyunBaiLianSpeechToText
 from models_provider.impl.aliyun_bai_lian_model_provider.model.tti import QwenTextToImageModel
@@ -80,6 +80,9 @@
                    ModelInfo('qwen-omni-turbo',
                              _('The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text.'),
                              ModelTypeConst.STT, aliyun_bai_lian_omi_stt_model_credential, AliyunBaiLianOmiSpeechToText),
+                   ModelInfo('qwen2.5-omni-7b',
+                             _('The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text.'),
+                             ModelTypeConst.STT, aliyun_bai_lian_omi_stt_model_credential, AliyunBaiLianOmiSpeechToText),
                    ]
 
 module_info_vl_list = [

…ian_model_provider/credential/omi_stt.py …an_model_provider/credential/omni_stt.pyapps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py renamed to apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omni_stt.py apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py renamed to apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omni_stt.py

@@ -17,7 +17,8 @@ class AliyunBaiLianOmiSTTModelParams(BaseForm):
 
 
 class AliyunBaiLianOmiSTTModelCredential(BaseForm, BaseModelCredential):
-    api_key = PasswordInputField("API key", required=True)
+    api_url = forms.TextInputField(_('API URL'), required=True)
+    api_key = forms.PasswordInputField(_('API Key'), required=True)
 
     def is_valid(self,
                  model_type: str,

…bai_lian_model_provider/model/omi_stt.py …ai_lian_model_provider/model/omni_stt.pyapps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py renamed to apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omni_stt.py apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py renamed to apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omni_stt.py

@@ -12,6 +12,7 @@
 
 class AliyunBaiLianOmiSpeechToText(MaxKBBaseModel, BaseSpeechToText):
     api_key: str
+    api_url: str
     model: str
     params: dict
 
@@ -20,6 +21,7 @@ def __init__(self, **kwargs):
         self.api_key = kwargs.get('api_key')
         self.model = kwargs.get('model')
         self.params = kwargs.get('params')
+        self.api_url = kwargs.get('api_url')
 
     @staticmethod
     def is_cache_model():
@@ -30,6 +32,7 @@ def new_instance(model_type, model_name, model_credential: Dict[str, object], **
         return AliyunBaiLianOmiSpeechToText(
             model=model_name,
             api_key=model_credential.get('api_key'),
+            api_url=model_credential.get('api_url') ,
             params= model_kwargs,
             **model_kwargs
         )
@@ -47,13 +50,13 @@ def speech_to_text(self, audio_file):
             client = OpenAI(
                 # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx",
                 api_key=self.api_key,
-                base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+                base_url=self.api_url,
             )
 
             base64_audio = base64.b64encode(audio_file.read()).decode("utf-8")
 
             completion = client.chat.completions.create(
-                model="qwen-omni-turbo-0119",
+                model=self.model,
                 messages=[
                     {
                         "role": "user",
@@ -71,16 +74,15 @@ def speech_to_text(self, audio_file):
                 ],
                 # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"]
                 modalities=["text"],
-                audio={"voice": "Cherry", "format": "mp3"},
                 # stream 必须设置为 True，否则会报错
                 stream=True,
                 stream_options={"include_usage": True},
             )
             result = []
             for chunk in completion:
-                if chunk.choices and hasattr(chunk.choices[0].delta, 'audio'):
-                    transcript = chunk.choices[0].delta.audio.get('transcript')
-                    result.append(transcript)
+                if chunk.choices and hasattr(chunk.choices[0].delta, 'content'):
+                    content = chunk.choices[0].delta.content
+                    result.append(content)
             return "".join(result)
 
         except Exception as err:

apps/models_provider/impl/aliyun_bai_lian_model_provider/model/stt.py

@@ -30,8 +30,6 @@ def new_instance(model_type, model_name, model_credential: Dict[str, object], **
             optional_params['max_tokens'] = model_kwargs['max_tokens']
         if 'temperature' in model_kwargs and model_kwargs['temperature'] is not None:
             optional_params['temperature'] = model_kwargs['temperature']
-        if model_name == 'qwen-omni-turbo':
-            optional_params['streaming'] = True
         return AliyunBaiLianSpeechToText(
             model=model_name,
             api_key=model_credential.get('api_key'),