fix: sorting (#508)

Author: iziang

kubechat/db/models.py

@@ -344,6 +344,9 @@ class CollectionSyncHistory(models.Model):
     start_time = models.DateTimeField()
     task_context = models.JSONField(default={})
     status = models.CharField(max_length=16, choices=CollectionSyncStatus.choices, default=CollectionSyncStatus.RUNNING)
+    gmt_created = models.DateTimeField(auto_now_add=True, null=True)
+    gmt_updated = models.DateTimeField(auto_now=True, null=True)
+    gmt_deleted = models.DateTimeField(null=True, blank=True)
 
     def update_execution_time(self):
         self.refresh_from_db()

kubechat/db/ops.py

@@ -38,20 +38,15 @@ class PagedResult(BaseModel):
 
 
 def build_pq(request) -> Optional[PagedQuery]:
-    # FIXME this is compatible with the old frontend, once the frontend is updated,
-    # FIXME we should do pagination and add default page_number and page_size
     page_number = request.GET.get('page_number')
     page_size = request.GET.get('page_size')
-    if not page_number or not page_size:
-        return None
-
     match_key = request.GET.get('match_key', "")
     match_value = request.GET.get('match_value', "")
     order_by = request.GET.get('order_by', "")
     order_desc = request.GET.get('order_desc', "true")
     return PagedQuery(
-        page_number=int(page_number),
-        page_size=int(page_size),
+        page_number=int(page_number) if page_number else None,
+        page_size=int(page_size) if page_size else None,
         match_key=match_key,
         match_value=match_value,
         order_by=order_by,
@@ -66,13 +61,13 @@ def build_order_by(pq: PagedQuery) -> str:
 
 
 async def build_pr(pq: PagedQuery, query_set: QuerySet) -> PagedResult:
-    if not pq:
-        return PagedResult(count=-1, data=query_set)
+    count = await get_count(query_set)
+    query_set = query_set.order_by(build_order_by(pq))
+    if not pq or not pq.page_number or not pq.page_size:
+        return PagedResult(count=count, page_number=0, page_size=count, data=query_set)
 
     offset = (pq.page_number - 1) * pq.page_size
     limit = pq.page_size
-    count = await get_count(query_set)
-    query_set = query_set.order_by(build_order_by(pq))
     return PagedResult(count=count, page_number=pq.page_number, page_size=pq.page_size,
                        data=query_set[offset:offset + limit])
 

…ota_userquota_alter_botintegration_id.py …ctionsynchistory_gmt_created_and_more.pykubechat/migrations/0007_quota_userquota_alter_botintegration_id.py renamed to kubechat/migrations/0007_quota_userquota_collectionsynchistory_gmt_created_and_more.py kubechat/migrations/0007_quota_userquota_alter_botintegration_id.py renamed to kubechat/migrations/0007_quota_userquota_collectionsynchistory_gmt_created_and_more.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.6 on 2023-12-01 15:19
+# Generated by Django 4.2.6 on 2023-12-02 03:18
 
 from django.db import migrations, models
 import kubechat.db.models
@@ -34,6 +34,21 @@ class Migration(migrations.Migration):
                 ('gmt_deleted', models.DateTimeField(blank=True, null=True)),
             ],
         ),
+        migrations.AddField(
+            model_name='collectionsynchistory',
+            name='gmt_created',
+            field=models.DateTimeField(auto_now_add=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='collectionsynchistory',
+            name='gmt_deleted',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='collectionsynchistory',
+            name='gmt_updated',
+            field=models.DateTimeField(auto_now=True, null=True),
+        ),
         migrations.AlterField(
             model_name='botintegration',
             name='id',

kubechat/tasks/sync_documents_task.py

@@ -113,7 +113,7 @@ def add_document(document):
             if document_limit and docs_count >= document_limit:
                 exceeded_limit_docs.append(src_doc)
                 continue
-            add_document(src_doc)
+            doc = add_document(src_doc)
             docs_count += 1
             tasks.append(add_index_for_document.s(doc.id))
 

readers/local_path_embedding.py

@@ -48,7 +48,7 @@ def load_data(self, **kwargs) -> Tuple[List[str], str, List]:
         sensitive_protect_method = kwargs.get('sensitive_protect_method', ProtectAction.WARNING_NOT_STORED)
         docs, file_name = self.reader.load_data()
         if not docs:
-            return []
+            return [], "", []
 
         nodes: List[Node] = []
         nodes_with_embedding: List[NodeWithEmbedding] = []