fix: 处理历史数据 (#1692)

Author: shaohuzhang1

apps/common/event/listener_manage.py

@@ -170,8 +170,7 @@ def get_aggregation_document_status(document_id):
         def aggregation_document_status():
             sql = get_file_content(
                 os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
-            native_update({'document_custom_sql': QuerySet(Document).filter(id=document_id),
-                           'default_sql': QuerySet(Document).filter(id=document_id)}, sql, with_table_name=True)
+            native_update({'document_custom_sql': QuerySet(Document).filter(id=document_id)}, sql, with_table_name=True)
 
         return aggregation_document_status
 
@@ -180,8 +179,7 @@ def get_aggregation_document_status_by_dataset_id(dataset_id):
         def aggregation_document_status():
             sql = get_file_content(
                 os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
-            native_update({'document_custom_sql': QuerySet(Document).filter(dataset_id=dataset_id),
-                           'default_sql': QuerySet(Document).filter(dataset_id=dataset_id)}, sql)
+            native_update({'document_custom_sql': QuerySet(Document).filter(dataset_id=dataset_id)}, sql)
 
         return aggregation_document_status
 
@@ -190,7 +188,7 @@ def get_aggregation_document_status_by_query_set(queryset):
         def aggregation_document_status():
             sql = get_file_content(
                 os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
-            native_update({'document_custom_sql': queryset, 'default_sql': queryset}, sql)
+            native_update({'document_custom_sql': queryset}, sql)
 
         return aggregation_document_status
 

apps/dataset/migrations/0011_document_status_meta_paragraph_status_meta_and_more.py

@@ -1,11 +1,23 @@
 # Generated by Django 4.2.15 on 2024-11-22 14:44
+from django.db.models import QuerySet
 
-import dataset.models.data_set
 from django.db import migrations, models
 
+import dataset
+from common.event import ListenerManagement
+from dataset.models import State, TaskType
 
-class Migration(migrations.Migration):
 
+def updateDocumentStatus(apps, schema_editor):
+    ParagraphModel = apps.get_model('dataset', 'Paragraph')
+    DocumentModel = apps.get_model('dataset', 'Document')
+    success_list = QuerySet(DocumentModel).filter(status='2')
+    ListenerManagement.update_status(QuerySet(ParagraphModel).filter(document_id__in=[d.id for d in success_list]),
+                                     TaskType.EMBEDDING, State.SUCCESS)
+    ListenerManagement.get_aggregation_document_status_by_query_set(QuerySet(DocumentModel))()
+
+
+class Migration(migrations.Migration):
     dependencies = [
         ('dataset', '0010_file_meta'),
     ]
@@ -31,4 +43,5 @@ class Migration(migrations.Migration):
             name='status',
             field=models.CharField(default=dataset.models.data_set.Status.__str__, max_length=20, verbose_name='状态'),
         ),
+        migrations.RunPython(updateDocumentStatus)
     ]

apps/dataset/sql/update_document_status_meta.sql

@@ -22,4 +22,4 @@ FROM
 	GROUP BY
 		document_id
 	) tmp
-${default_sql}
+WHERE "document".id="tmp".document_id

ui/src/utils/status.ts

@@ -65,4 +65,4 @@ class Status {
     return r.reverse().join('')
   }
 }
-export { Status, State, TaskType, TaskTypeInterface, StateInterface }
+export { Status, State, TaskType, type TaskTypeInterface, type StateInterface }

ui/src/views/document/index.vue

@@ -381,7 +381,7 @@ const {
 
 const { common, dataset, document } = useStore()
 const storeKey = 'documents'
-const getTaskState = (status, taskType) => {
+const getTaskState = (status: string, taskType: number) => {
   const statusList = status.split('').reverse()
   return taskType - 1 > statusList.length + 1 ? 'n' : statusList[taskType - 1]
 }