Pr@main@fix bugs (#27)

* fix: 优化word分段规则

* fix: 去除标题特殊字符

* fix: 对话重新生成问题

---------

Co-authored-by: wangdan-fit2cloud <[email protected]>

Author: shaohuzhang1

apps/common/handle/impl/doc_split_handle.py

@@ -22,11 +22,25 @@
 
 
 class DocSplitHandle(BaseSplitHandle):
+    @staticmethod
+    def paragraph_to_md(paragraph):
+        psn = paragraph.style.name
+        if psn.startswith('Heading'):
+            try:
+                return "".join(["#" for i in range(int(psn.replace("Heading ", '')))]) + " " + paragraph.text
+            except Exception as e:
+                return paragraph.text
+        return paragraph.text
+
+    def to_md(self, doc):
+        ps = doc.paragraphs
+        return "\n".join([self.paragraph_to_md(para) for para in ps])
+
     def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer):
         try:
             buffer = get_buffer(file)
             doc = Document(io.BytesIO(buffer))
-            content = "\n".join([para.text for para in doc.paragraphs])
+            content = self.to_md(doc)
             if pattern_list is not None and len(pattern_list) > 0:
                 split_model = SplitModel(pattern_list, with_filter, limit)
             else:

apps/common/util/split_model.py

@@ -351,9 +351,15 @@ def sub_title(paragraph: Dict):
 
     @staticmethod
     def filter_title_special_characters(paragraph: Dict):
-        return {**paragraph, 'title': paragraph.get('title').replace("#", '') if 'title' in paragraph else ''}
+        title = paragraph.get('title') if 'title' in paragraph else ''
+        for title_special_characters in title_special_characters_list:
+            title = title.replace(title_special_characters, '')
+        return {**paragraph,
+                'title': title}
 
 
+title_special_characters_list = ['#', '\n', '\r', '\\s']
+
 default_split_pattern = {
     'md': [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#)## (?!#).*'), re.compile("(?<!#)### (?!#).*"),
            re.compile("(?<!#)#### (?!#).*"), re.compile("(?<!#)##### (?!#).*"),

ui/src/api/application.ts

@@ -166,14 +166,12 @@ const getChatOpen: (applicaiton_id: String) => Promise<Result<any>> = (applicait
 }
 /**
  * 对话
- * @param 参数 
+ * @param 参数
  * chat_id: string
- * {
-    "message": "string",
-  }
+ * data
  */
-const postChatMessage: (chat_id: string, message: string) => Promise<any> = (chat_id, message) => {
-  return postStream(`/api${prefix}/chat_message/${chat_id}`, { message })
+const postChatMessage: (chat_id: string, data: any) => Promise<any> = (chat_id, message) => {
+  return postStream(`/api${prefix}/chat_message/${chat_id}`, data)
 }
 
 /**

ui/src/components/ai-chat/index.vue

@@ -285,7 +285,7 @@ function sendChatHandle(event: any) {
   if (!event.ctrlKey) {
     // 如果没有按下组合键ctrl，则会阻止默认事件
     event.preventDefault()
-    if (!isDisabledChart.value && !loading.value&&!event.isComposing) {
+    if (!isDisabledChart.value && !loading.value && !event.isComposing) {
       chatMessage()
     }
   } else {
@@ -418,7 +418,7 @@ const errorWrite = (chat: any, message?: string) => {
   ChatManagement.append(chat.id, message || '抱歉，当前正在维护，无法提供服务，请稍后再试！')
   ChatManagement.close(chat.id)
 }
-function chatMessage(chat?: any, problem?: string) {
+function chatMessage(chat?: any, problem?: string, re_chat?: boolean) {
   loading.value = true
   if (!chat) {
     chat = reactive({
@@ -443,9 +443,13 @@ function chatMessage(chat?: any, problem?: string) {
       errorWrite(chat)
     })
   } else {
+    const obj = {
+      message: chat.problem_text,
+      re_chat: re_chat || false
+    }
     // 对话
     applicationApi
-      .postChatMessage(chartOpenId.value, chat.problem_text)
+      .postChatMessage(chartOpenId.value, obj)
       .then((response) => {
         if (response.status === 401) {
           application
@@ -491,7 +495,7 @@ function chatMessage(chat?: any, problem?: string) {
 
 function regenerationChart(item: chatType) {
   inputValue.value = item.problem_text
-  chatMessage()
+  chatMessage(null, '', true)
 }
 
 function getSourceDetail(row: any) {

ui/src/views/applicaiton-overview/index.vue

@@ -197,8 +197,8 @@ function getAppStatistics() {
 
 function refreshAccessToken() {
   MsgConfirm(
-    `是否重新生成公共访问链接?`,
-    `重新生成公共访问链接会影响嵌入第三方脚本变更，需要将新脚本重新嵌入第三方，请谨慎操作！`,
+    `是否重新生成公开访问链接?`,
+    `重新生成公开访问链接会影响嵌入第三方脚本变更，需要将新脚本重新嵌入第三方，请谨慎操作！`,
     {
       confirmButtonText: '确认'
     }

ui/src/views/dataset/component/UploadComponent.vue

@@ -27,7 +27,7 @@
             <em> 选择文件上传 </em>
           </p>
           <div class="upload__decoration">
-            <p>支持格式：TXT、Markdown，每次最多上传50个文件，每个文件不超过 10MB</p>
+            <p>支持格式：TXT、Markdown、PDF、DOC、DOCX，每次最多上传50个文件，每个文件不超过 10MB</p>
             <p>若使用【高级分段】建议上传前规范文件的分段标识</p>
           </div>
         </div>