Skip to content

Commit 93a5c6e

Browse files
shaohuzhang1shaohuzhang1
authored
fix: 修复文档状态数据数据错误问题 (#1697)
1 parent 5257536 commit 93a5c6e

File tree

6 files changed

+130
-110
lines changed

6 files changed

+130
-110
lines changed

‎apps/common/db/sql_execute.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,9 @@ def update_execute(sql: str, params):
3636
"""
3737
with connection.cursor() as cursor:
3838
cursor.execute(sql, params)
39+
affected_rows = cursor.rowcount
3940
cursor.close()
40-
return None
41+
return affected_rows
4142

4243

4344
def select_list(sql: str, params: List):

‎apps/common/event/listener_manage.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,12 @@
1010
import logging
1111
import os
1212
import threading
13+
import time
1314
import traceback
1415
from typing import List
1516

1617
import django.db.models
17-
from django.db import models
18+
from django.db import models, transaction
1819
from django.db.models import QuerySet
1920
from django.db.models.functions import Substr, Reverse
2021
from langchain_core.embeddings import Embeddings
@@ -168,6 +169,7 @@ def embedding_paragraph_apply(paragraph_list):
168169
@staticmethod
169170
def get_aggregation_document_status(document_id):
170171
def aggregation_document_status():
172+
pass
171173
sql = get_file_content(
172174
os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
173175
native_update({'document_custom_sql': QuerySet(Document).filter(id=document_id)}, sql, with_table_name=True)

‎apps/common/util/page_utils.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ def page(query_set, page_size, handler, is_the_task_interrupted=lambda: False):
1818
@param is_the_task_interrupted: 任务是否被中断
1919
@return:
2020
"""
21+
query = query_set.order_by("id")
2122
count = query_set.count()
2223
for i in range(0, ceil(count / page_size)):
2324
if is_the_task_interrupted():
2425
return
2526
offset = i * page_size
26-
paragraph_list = query_set[offset: offset + page_size]
27+
paragraph_list = query.all()[offset: offset + page_size]
2728
handler(paragraph_list)

‎apps/dataset/serializers/document_serializers.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -613,7 +613,8 @@ def refresh(self, with_valid=True):
613613
document_id = self.data.get("document_id")
614614
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
615615
State.PENDING)
616-
ListenerManagement.update_status(QuerySet(Paragraph).filter(document_id=document_id), TaskType.EMBEDDING,
616+
ListenerManagement.update_status(QuerySet(Paragraph).filter(document_id=document_id),
617+
TaskType.EMBEDDING,
617618
State.PENDING)
618619
ListenerManagement.get_aggregation_document_status(document_id)()
619620
embedding_model_id = get_embedding_model_id_by_dataset_id(dataset_id=self.data.get('dataset_id'))
@@ -708,8 +709,8 @@ def is_valid(self, *, raise_exception=False):
708709

709710
@staticmethod
710711
def post_embedding(result, document_id, dataset_id):
711-
model_id = get_embedding_model_id_by_dataset_id(dataset_id)
712-
embedding_by_document.delay(document_id, model_id)
712+
DocumentSerializers.Operate(
713+
data={'dataset_id': dataset_id, 'document_id': document_id}).refresh()
713714
return result
714715

715716
@staticmethod
@@ -907,8 +908,8 @@ def get_request_body_api():
907908
@staticmethod
908909
def post_embedding(document_list, dataset_id):
909910
for document_dict in document_list:
910-
model_id = get_embedding_model_id_by_dataset_id(dataset_id)
911-
embedding_by_document.delay(document_dict.get('id'), model_id)
911+
DocumentSerializers.Operate(
912+
data={'dataset_id': dataset_id, 'document_id': document_dict.get('id')}).refresh()
912913
return document_list
913914

914915
@post(post_function=post_embedding)

‎ui/src/views/document/component/Status.vue‎

Lines changed: 13 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,13 @@
11
<template>
2-
<el-popover placement="top" :width="450" trigger="hover">
3-
<template #default>
4-
<el-row :gutter="3" v-for="status in statusTable" :key="status.type">
5-
<el-col :span="4">{{ taskTypeMap[status.type] }} </el-col>
6-
<el-col :span="4">
7-
<el-text v-if="status.state === State.SUCCESS || status.state === State.REVOKED">
8-
<el-icon class="success"><SuccessFilled /></el-icon>
9-
{{ stateMap[status.state](status.type) }}
10-
</el-text>
11-
<el-text v-else-if="status.state === State.FAILURE">
12-
<el-icon class="danger"><CircleCloseFilled /></el-icon>
13-
{{ stateMap[status.state](status.type) }}
14-
</el-text>
15-
<el-text v-else-if="status.state === State.STARTED">
16-
<el-icon class="is-loading primary"><Loading /></el-icon>
17-
{{ stateMap[status.state](status.type) }}
18-
</el-text>
19-
<el-text v-else-if="status.state === State.PENDING">
20-
<el-icon class="is-loading primary"><Loading /></el-icon>
21-
{{ stateMap[status.state](status.type) }}
22-
</el-text>
23-
<el-text v-else-if="aggStatus?.value === State.REVOKE">
24-
<el-icon class="is-loading primary"><Loading /></el-icon>
25-
{{ stateMap[aggStatus.value](aggStatus.key) }}
26-
</el-text>
27-
</el-col>
28-
<el-col :span="5">
29-
完成
30-
{{
31-
Object.keys(status.aggs ? status.aggs : {})
32-
.filter((k) => k == State.SUCCESS)
33-
.map((k) => status.aggs[k])
34-
.reduce((x: any, y: any) => x + y, 0)
35-
}}/{{
36-
Object.values(status.aggs ? status.aggs : {}).reduce((x: any, y: any) => x + y, 0)
37-
}}
38-
</el-col>
39-
<el-col :span="9">
40-
{{
41-
status.time
42-
? status.time[
43-
status.state == State.REVOKED ? State.REVOKED : State.PENDING
44-
]?.substring(0, 19)
45-
: undefined
46-
}}
47-
</el-col>
48-
</el-row>
2+
<el-popover v-model:visible="visible" placement="top" :width="450" trigger="hover">
3+
<template #default
4+
><StatusTable
5+
v-if="visible"
6+
:status="status"
7+
:statusMeta="statusMeta"
8+
:taskTypeMap="taskTypeMap"
9+
:stateMap="stateMap"
10+
></StatusTable>
4911
</template>
5012
<template #reference>
5113
<el-text v-if="aggStatus?.value === State.SUCCESS || aggStatus?.value === State.REVOKED">
@@ -72,11 +34,11 @@
7234
</el-popover>
7335
</template>
7436
<script setup lang="ts">
75-
import { computed } from 'vue'
76-
import { Status, TaskType, State, type TaskTypeInterface } from '@/utils/status'
77-
import { mergeWith } from 'lodash'
37+
import { computed, ref } from 'vue'
38+
import { TaskType, State } from '@/utils/status'
39+
import StatusTable from '@/views/document/component/StatusTable.vue'
7840
const props = defineProps<{ status: string; statusMeta: any }>()
79-
41+
const visible = ref<boolean>(false)
8042
const checkList: Array<string> = [
8143
State.REVOKE,
8244
State.STARTED,
@@ -112,56 +74,5 @@ const stateMap: any = {
11274
[State.FAILURE]: (type: number) => '失败',
11375
[State.SUCCESS]: (type: number) => '成功'
11476
}
115-
116-
const parseAgg = (agg: { count: number; status: string }) => {
117-
const status = new Status(agg.status)
118-
return Object.keys(TaskType)
119-
.map((key) => {
120-
const value = TaskType[key as keyof TaskTypeInterface]
121-
return { [value]: { [status.task_status[value]]: agg.count } }
122-
})
123-
.reduce((x, y) => ({ ...x, ...y }), {})
124-
}
125-
126-
const customizer: (x: any, y: any) => any = (objValue: any, srcValue: any) => {
127-
if (objValue == undefined && srcValue) {
128-
return srcValue
129-
}
130-
if (srcValue == undefined && objValue) {
131-
return objValue
132-
}
133-
// 如果是数组，我们将元素进行聚合
134-
if (typeof objValue === 'object' && typeof srcValue === 'object') {
135-
// 若是object类型的对象，我们进行递归
136-
return mergeWith(objValue, srcValue, customizer)
137-
} else {
138-
// 否则，单纯的将值进行累加
139-
return objValue + srcValue
140-
}
141-
}
142-
const aggs = computed(() => {
143-
return (props.statusMeta.aggs ? props.statusMeta.aggs : [])
144-
.map((agg: any) => {
145-
return parseAgg(agg)
146-
})
147-
.reduce((x: any, y: any) => {
148-
return mergeWith(x, y, customizer)
149-
}, {})
150-
})
151-
152-
const statusTable = computed(() => {
153-
return Object.keys(TaskType)
154-
.map((key) => {
155-
const value = TaskType[key as keyof TaskTypeInterface]
156-
const parseStatus = new Status(props.status)
157-
return {
158-
type: value,
159-
state: parseStatus.task_status[value],
160-
aggs: aggs.value[value],
161-
time: props.statusMeta.state_time[value]
162-
}
163-
})
164-
.filter((item) => item.state !== State.IGNORED)
165-
})
16677
</script>
16778
<style lang="scss" scoped></style>

‎ui/src/views/document/component/StatusTable.vue‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
<template>
2+
<el-row :gutter="3" v-for="status in statusTable" :key="status.type">
3+
<el-col :span="4">{{ taskTypeMap[status.type] }} </el-col>
4+
<el-col :span="4">
5+
<el-text v-if="status.state === State.SUCCESS || status.state === State.REVOKED">
6+
<el-icon class="success"><SuccessFilled /></el-icon>
7+
{{ stateMap[status.state](status.type) }}
8+
</el-text>
9+
<el-text v-else-if="status.state === State.FAILURE">
10+
<el-icon class="danger"><CircleCloseFilled /></el-icon>
11+
{{ stateMap[status.state](status.type) }}
12+
</el-text>
13+
<el-text v-else-if="status.state === State.STARTED">
14+
<el-icon class="is-loading primary"><Loading /></el-icon>
15+
{{ stateMap[status.state](status.type) }}
16+
</el-text>
17+
<el-text v-else-if="status.state === State.PENDING">
18+
<el-icon class="is-loading primary"><Loading /></el-icon>
19+
{{ stateMap[status.state](status.type) }}
20+
</el-text>
21+
<el-text v-else-if="status.state === State.REVOKE">
22+
<el-icon class="is-loading primary"><Loading /></el-icon>
23+
{{ stateMap[status.state](status.type) }}
24+
</el-text>
25+
</el-col>
26+
<el-col :span="7">
27+
完成
28+
{{
29+
Object.keys(status.aggs ? status.aggs : {})
30+
.filter((k) => k == State.SUCCESS)
31+
.map((k) => status.aggs[k])
32+
.reduce((x: any, y: any) => x + y, 0)
33+
}}/{{ Object.values(status.aggs ? status.aggs : {}).reduce((x: any, y: any) => x + y, 0) }}
34+
</el-col>
35+
<el-col :span="9">
36+
{{
37+
status.time
38+
? status.time[status.state == State.REVOKED ? State.REVOKED : State.PENDING]?.substring(
39+
0,
40+
19
41+
)
42+
: undefined
43+
}}
44+
</el-col>
45+
</el-row>
46+
</template>
47+
<script setup lang="ts">
48+
import { computed } from 'vue'
49+
import { Status, TaskType, State, type TaskTypeInterface } from '@/utils/status'
50+
import { mergeWith } from 'lodash'
51+
const props = defineProps<{ status: string; statusMeta: any; stateMap: any; taskTypeMap: any }>()
52+
53+
const parseAgg = (agg: { count: number; status: string }) => {
54+
const status = new Status(agg.status)
55+
return Object.keys(TaskType)
56+
.map((key) => {
57+
const value = TaskType[key as keyof TaskTypeInterface]
58+
return { [value]: { [status.task_status[value]]: agg.count } }
59+
})
60+
.reduce((x, y) => ({ ...x, ...y }), {})
61+
}
62+
63+
const customizer: (x: any, y: any) => any = (objValue: any, srcValue: any) => {
64+
if (objValue == undefined && srcValue) {
65+
return srcValue
66+
}
67+
if (srcValue == undefined && objValue) {
68+
return objValue
69+
}
70+
// 如果是数组，我们将元素进行聚合
71+
if (typeof objValue === 'object' && typeof srcValue === 'object') {
72+
// 若是object类型的对象，我们进行递归
73+
return mergeWith(objValue, srcValue, customizer)
74+
} else {
75+
// 否则，单纯的将值进行累加
76+
return objValue + srcValue
77+
}
78+
}
79+
const aggs = computed(() => {
80+
return (props.statusMeta.aggs ? props.statusMeta.aggs : [])
81+
.map((agg: any) => {
82+
return parseAgg(agg)
83+
})
84+
.reduce((x: any, y: any) => {
85+
return mergeWith(x, y, customizer)
86+
}, {})
87+
})
88+
89+
const statusTable = computed(() => {
90+
return Object.keys(TaskType)
91+
.map((key) => {
92+
const value = TaskType[key as keyof TaskTypeInterface]
93+
const parseStatus = new Status(props.status)
94+
return {
95+
type: value,
96+
state: parseStatus.task_status[value],
97+
aggs: aggs.value[value],
98+
time: props.statusMeta.state_time[value]
99+
}
100+
})
101+
.filter((item) => item.state !== State.IGNORED)
102+
})
103+
</script>
104+
<style lang="scss"></style>

0 commit comments

Comments
 (0)