Merge branch '1Panel-dev:main' into main

Author: litongjava

apps/application/flow/step_node/application_node/i_application_node.py

@@ -12,6 +12,8 @@ class ApplicationNodeSerializer(serializers.Serializer):
     question_reference_address = serializers.ListField(required=True, error_messages=ErrMessage.list("用户问题"))
     api_input_field_list = serializers.ListField(required=False, error_messages=ErrMessage.list("api输入字段"))
     user_input_field_list = serializers.ListField(required=False, error_messages=ErrMessage.uuid("用户输入字段"))
+    image_list = serializers.ListField(required=False, error_messages=ErrMessage.list("图片"))
+    document_list = serializers.ListField(required=False, error_messages=ErrMessage.list("文档"))
 
 
 class IApplicationNode(INode):
@@ -31,10 +33,27 @@ def _run(self):
         for user_input_field in self.node_params_serializer.data.get('user_input_field_list', []):
             kwargs[user_input_field['field']] = self.workflow_manage.get_reference_field(user_input_field['value'][0],
                                                                                          user_input_field['value'][1:])
-
+        # 判断是否包含这个属性
+        app_document_list = self.node_params_serializer.data.get('document_list', [])
+        if app_document_list and len(app_document_list) > 0:
+            app_document_list = self.workflow_manage.get_reference_field(
+                app_document_list[0],
+                app_document_list[1:])
+            for document in app_document_list:
+                if 'file_id' not in document:
+                    raise ValueError("参数值错误: 上传的文档中缺少file_id")
+        app_image_list = self.node_params_serializer.data.get('image_list', [])
+        if app_image_list and len(app_image_list) > 0:
+            app_image_list = self.workflow_manage.get_reference_field(
+                app_image_list[0],
+                app_image_list[1:])
+            for image in app_image_list:
+                if 'file_id' not in image:
+                    raise ValueError("参数值错误: 上传的图片中缺少file_id")
         return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data,
+                            app_document_list=app_document_list, app_image_list=app_image_list,
                             message=str(question), **kwargs)
 
     def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type,
-                **kwargs) -> NodeResult:
+                app_document_list=None, app_image_list=None, **kwargs) -> NodeResult:
         pass

apps/application/flow/step_node/application_node/impl/base_application_node.py

@@ -71,6 +71,7 @@ def save_context(self, details, workflow_manage):
         self.answer_text = details.get('answer')
 
     def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type,
+                app_document_list=None, app_image_list=None,
                 **kwargs) -> NodeResult:
         from application.serializers.chat_message_serializers import ChatMessageSerializer
         # 生成嵌入应用的chat_id
@@ -79,13 +80,20 @@ def execute(self, application_id, message, chat_id, chat_record_id, stream, re_c
             'application_id': application_id,
             'abstract': message
         })
+        if app_document_list is None:
+            app_document_list = []
+        if app_image_list is None:
+            app_image_list = []
         response = ChatMessageSerializer(
             data={'chat_id': current_chat_id, 'message': message,
                   're_chat': re_chat,
                   'stream': stream,
                   'application_id': application_id,
                   'client_id': client_id,
-                  'client_type': client_type, 'form_data': kwargs}).chat(base_to_response=OpenaiToResponse())
+                  'client_type': client_type,
+                  'document_list': app_document_list,
+                  'image_list': app_image_list,
+                  'form_data': kwargs}).chat(base_to_response=OpenaiToResponse())
         if response.status_code == 200:
             if stream:
                 content_generator = response.streaming_content
@@ -126,5 +134,7 @@ def get_details(self, index: int, **kwargs):
             'answer_tokens': self.context.get('answer_tokens'),
             'status': self.status,
             'err_message': self.err_message,
-            'global_fields': global_fields
+            'global_fields': global_fields,
+            'document_list': self.workflow_manage.document_list,
+            'image_list': self.workflow_manage.image_list
         }

apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py

@@ -116,13 +116,17 @@ def generate_prompt_question(self, prompt):
 
     def generate_message_list(self, image_model, system: str, prompt: str, history_message, image):
         if image is not None and len(image) > 0:
-            file_id = image[0]['file_id']
-            file = QuerySet(File).filter(id=file_id).first()
-            base64_image = base64.b64encode(file.get_byte()).decode("utf-8")
+            # 处理多张图片
+            images = []
+            for img in image:
+                file_id = img['file_id']
+                file = QuerySet(File).filter(id=file_id).first()
+                base64_image = base64.b64encode(file.get_byte()).decode("utf-8")
+                images.append({'type': 'image_url', 'image_url': {'url': f'data:image/jpeg;base64,{base64_image}'}})
             messages = [HumanMessage(
                 content=[
                     {'type': 'text', 'text': self.workflow_manage.generate_prompt(prompt)},
-                    {'type': 'image_url', 'image_url': {'url': f'data:image/jpeg;base64,{base64_image}'}},
+                    *images
                 ])]
         else:
             messages = [HumanMessage(self.workflow_manage.generate_prompt(prompt))]

apps/application/flow/step_node/start_node/impl/base_start_node.py

@@ -77,5 +77,6 @@ def get_details(self, index: int, **kwargs):
             'status': self.status,
             'err_message': self.err_message,
             'image_list': self.context.get('image'),
+            'document_list': self.context.get('document'),
             'global_fields': global_fields
         }

apps/application/flow/workflow_manage.py

@@ -52,7 +52,7 @@ def __init__(self, _id: str, _type: str, x: int, y: int, properties: dict, **kwa
             self.__setattr__(keyword, kwargs.get(keyword))
 
 
-end_nodes = ['ai-chat-node', 'reply-node', 'function-node', 'function-lib-node', 'application-node']
+end_nodes = ['ai-chat-node', 'reply-node', 'function-node', 'function-lib-node', 'application-node', 'image-understand-node']
 
 
 class Flow:

apps/application/serializers/chat_serializers.py

@@ -38,7 +38,7 @@
 from common.util.file_util import get_file_content
 from common.util.lock import try_lock, un_lock
 from dataset.models import Document, Problem, Paragraph, ProblemParagraphMapping
-from dataset.serializers.common_serializers import get_embedding_model_id_by_dataset_id
+from dataset.serializers.common_serializers import get_embedding_model_id_by_dataset_id, update_document_char_length
 from dataset.serializers.paragraph_serializers import ParagraphSerializers
 from embedding.task import embedding_by_paragraph, embedding_by_paragraph_list
 from setting.models import Model
@@ -620,6 +620,7 @@ def improve(self, instance: Dict, with_valid=True):
             # 插入关联问题
             problem_paragraph_mapping.save()
             chat_record.improve_paragraph_id_list.append(paragraph.id)
+            update_document_char_length(document_id)
             # 添加标注
             chat_record.save()
             return ChatRecordSerializerModel(chat_record).data, paragraph.id, dataset_id
@@ -718,5 +719,6 @@ def post_improve(self, instance: Dict):
 
             # 批量保存聊天记录
             ChatRecord.objects.bulk_update(chat_record_list, ['improve_paragraph_id_list'])
+            update_document_char_length(document_id)
 
             return paragraph_ids, dataset_id

apps/setting/models_provider/impl/openai_model_provider/credential/image.py

@@ -29,7 +29,9 @@ def is_valid(self, model_type: str, model_name, model_credential: Dict[str, obje
                     return False
         try:
             model = provider.get_model(model_type, model_name, model_credential)
-            model.stream([HumanMessage(content=[{"type": "text", "text": "你好"}])])
+            res = model.stream([HumanMessage(content=[{"type": "text", "text": "你好"}])])
+            for chunk in res:
+                print(chunk)
         except Exception as e:
             if isinstance(e, AppApiException):
                 raise e

apps/setting/models_provider/impl/openai_model_provider/model/image.py

@@ -17,9 +17,10 @@ class OpenAIImage(MaxKBBaseModel, ChatOpenAI):
     def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
         optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
         return OpenAIImage(
-            model=model_name,
+            model_name=model_name,
             openai_api_base=model_credential.get('api_base'),
             openai_api_key=model_credential.get('api_key'),
-            stream_options={"include_usage": True},
+            # stream_options={"include_usage": True},
+            streaming=True,
             **optional_params,
         )

apps/setting/models_provider/impl/qwen_model_provider/credential/image.py

@@ -50,7 +50,9 @@ def is_valid(self, model_type: str, model_name, model_credential: Dict[str, obje
                     return False
         try:
             model = provider.get_model(model_type, model_name, model_credential)
-            model.stream([HumanMessage(content=[{"type": "text", "text": "你好"}])])
+            res = model.stream([HumanMessage(content=[{"type": "text", "text": "你好"}])])
+            for chunk in res:
+                print(chunk)
         except Exception as e:
             if isinstance(e, AppApiException):
                 raise e

apps/setting/models_provider/impl/qwen_model_provider/model/image.py

@@ -13,10 +13,11 @@ class QwenVLChatModel(MaxKBBaseModel, ChatOpenAI):
     def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
         optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
         chat_tong_yi = QwenVLChatModel(
-            model=model_name,
+            model_name=model_name,
             openai_api_key=model_credential.get('api_key'),
             openai_api_base='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            stream_options={"include_usage": True},
+            # stream_options={"include_usage": True},
+            streaming=True,
             model_kwargs=optional_params,
         )
         return chat_tong_yi