Modifications post PR review

Author: Your Name

sde_collections/models/collection.py

@@ -634,22 +634,6 @@ def apply_all_patterns(self):
         for pattern in self.deltadivisionpatterns.all():
             pattern.apply()
 
-    def count_curated_urls(self):
-        """Return the count of Curated URLs for the collection."""
-        return CuratedUrl.objects.filter(collection=self).count()
-
-    def count_dump_urls(self):
-        """Return the count of all Dump URLs for the collection."""
-        return DumpUrl.objects.filter(collection=self).count()
-
-    def count_delta_urls(self):
-        """Return the count of Delta URLs identified."""
-        return DeltaUrl.objects.filter(collection=self).count()
-
-    def count_marked_for_deletion_urls(self):
-        """Return the count of Delta URLs marked for deletion."""
-        return DeltaUrl.objects.filter(collection=self, to_delete=True).count()
-
     def save(self, *args, **kwargs):
         # Call the function to generate the value for the generated_field based on the original_field
         if not self.config_folder:

sde_collections/sinequa_api.py

@@ -257,7 +257,7 @@ def get_full_texts(
                 if total_count is None:
                     total_count = response.get("TotalRowCount", 0)
 
-                yield self._process_rows_to_records(rows)
+                yield (self._process_rows_to_records(rows),total_count)
 
                 current_offset += len(rows)
 

sde_collections/tasks.py

@@ -16,7 +16,7 @@
 )
 from sde_collections.utils import slack_utils
 
-from .models.delta_url import DumpUrl
+from .models.delta_url import DumpUrl,DeltaUrl,CuratedUrl
 from .sinequa_api import Api
 from .utils.github_helper import GitHubHandler
 
@@ -174,11 +174,13 @@ def fetch_and_replace_full_text(collection_id, server_name):
     # Step 1: Delete existing DumpUrl entries
     deleted_count, _ = DumpUrl.objects.filter(collection=collection).delete()
     print(f"Deleted {deleted_count} old records.")
-
+    total_server_count=0
     try:
         # Step 2: Process data in batches
         total_processed = 0
-        for batch in api.get_full_texts(collection.config_folder):
+        for batch,total_count in api.get_full_texts(collection.config_folder):
+            if total_server_count == 0:
+                total_server_count = total_count
             with transaction.atomic():
                 DumpUrl.objects.bulk_create(
                     [
@@ -194,9 +196,15 @@ def fetch_and_replace_full_text(collection_id, server_name):
             total_processed += len(batch)
             print(f"Processed batch of {len(batch)} records. Total: {total_processed}")
 
+        dump_count = DumpUrl.objects.filter(collection=collection).count()
+
         # Step 3: Migrate dump URLs to delta URLs
         collection.migrate_dump_to_delta()
 
+        curated_count = CuratedUrl.objects.filter(collection=collection).count()
+        delta_count = DeltaUrl.objects.filter(collection=collection).count()
+        marked_for_deletion_count = DeltaUrl.objects.filter(collection=collection, to_delete=True).count()
+
         # Step 4: Update statuses if needed
         collection.refresh_from_db()
 
@@ -216,13 +224,8 @@ def fetch_and_replace_full_text(collection_id, server_name):
             collection.reindexing_status = ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION
             collection.save()
 
-        curated_count = collection.count_curated_urls()
-        dump_count = collection.count_dump_urls()
-        delta_count = collection.count_delta_urls()
-        deletion_count = collection.count_marked_for_deletion_urls()
-
         slack_utils.send_detailed_import_notification(
-            collection.name, total_processed, curated_count, dump_count, delta_count, deletion_count
+            collection.name, total_server_count, curated_count, dump_count, delta_count, marked_for_deletion_count
         )
         return f"Successfully processed {total_processed} records and updated the database."
 

sde_collections/utils/slack_utils.py

@@ -60,12 +60,12 @@ def format_slack_message(name, details, collection_id):
 
 
 def send_detailed_import_notification(
-    collection_name, total_processed, curated_count, dump_count, delta_count, deletion_count
+    collection_name, total_server_count, curated_count, dump_count, delta_count, marked_for_deletion_count
 ):
     message = (
         f"'{collection_name}' brought into COSMOS. "
-        f"Prior Curated: {curated_count}, URL counts - [Server: {total_processed}, "
-        f"Dump: {dump_count}, New Deltas: {delta_count}, Deleted: {deletion_count}]"
+        f"Prior Curated: {curated_count}, URL Counts - [Server: {total_server_count}, "
+        f"URLs Imported: {dump_count}, New Deltas: {delta_count}, Marked For Deletion: {marked_for_deletion_count}]"
     )
 
     webhook_url = settings.SLACK_WEBHOOK_URL