remove deprecated tasks.py code

Author: CarsonDavis

sde_collections/tasks.py

@@ -6,12 +6,13 @@
 from django.apps import apps
 from django.conf import settings
 from django.core import management
+from django.core.management.commands import loaddata
 from django.db import IntegrityError
 
 from config import celery_app
 
 from .models.collection import Collection, WorkflowStatusChoices
-from .models.delta_url import CuratedUrl, DeltaUrl, DumpUrl
+from .models.delta_url import DumpUrl
 from .sinequa_api import Api
 from .utils.github_helper import GitHubHandler
 
@@ -63,89 +64,6 @@ def _get_data_to_import(collection, server_name):
     return data_to_import
 
 
-def _compare_and_populate_delta_urls(collection):
-    """Compare DumpUrl and CuratedUrl and populate DeltaUrl."""
-    dump_urls = DumpUrl.objects.filter(collection=collection)
-    curated_urls = CuratedUrl.objects.filter(collection=collection)
-
-    DeltaUrl.objects.filter(collection=collection).delete()
-
-    curated_urls_dict = {url.url: url for url in curated_urls}
-
-    # Iterate over Dump URLs to find deltas
-    for dump_url in dump_urls:
-        curated_url = curated_urls_dict.get(dump_url.url)
-
-        if not curated_url:
-            # New URL found, add to DeltaUrl
-            DeltaUrl.objects.create(
-                collection=collection,
-                url=dump_url.url,
-                scraped_title=dump_url.scraped_title,
-                generated_title=dump_url.generated_title,
-                document_type=dump_url.document_type,
-                division=dump_url.division,
-                delete=False,
-            )
-        elif (
-            curated_url.scraped_title != dump_url.scraped_title
-            or curated_url.generated_title != dump_url.generated_title
-            or curated_url.document_type != dump_url.document_type
-            or curated_url.division != dump_url.division
-        ):
-            # Metadata changed, add to DeltaUrl
-            DeltaUrl.objects.create(
-                collection=collection,
-                url=dump_url.url,
-                scraped_title=dump_url.scraped_title,
-                generated_title=dump_url.generated_title,
-                document_type=dump_url.document_type,
-                division=dump_url.division,
-                delete=False,
-            )
-
-    # Mark any missing URLs in CuratedUrl as deleted in DeltaUrl
-    dump_url_set = set(dump_urls.values_list("url", flat=True))
-    for curated_url in curated_urls:
-        if curated_url.url not in dump_url_set:
-            DeltaUrl.objects.create(
-                collection=collection,
-                url=curated_url.url,
-                scraped_title=curated_url.scraped_title,
-                generated_title=curated_url.generated_title,
-                document_type=curated_url.document_type,
-                division=curated_url.division,
-                delete=True,
-            )
-
-
-# TODO: Bishwas wrote this but it is outdated.
-# def populate_dump_urls(collection):
-#     urls = Url.objects.filter(collection=collection)
-
-#     for url_instance in urls:
-#         try:
-#             # Create DumpUrl by passing in the parent Url fields
-#             dump_url_instance = DumpUrl(
-#                 id=url_instance.id,
-#                 collection=url_instance.collection,
-#                 url=url_instance.url,
-#                 scraped_title=url_instance.scraped_title,
-#                 visited=url_instance.visited,
-#                 document_type=url_instance.document_type,
-#                 division=url_instance.division,
-#             )
-#             dump_url_instance.save()  # Save both Url and DumpUrl entries
-
-#             print(f"Created DumpUrl: {dump_url_instance.url} - {dump_url_instance.scraped_title}")
-
-#         except Exception as e:
-#             print(f"Error creating DumpUrl for {url_instance.url}: {str(e)}")
-#             continue
-
-#     print(f"Successfully populated DumpUrl model with {urls.count()} entries.")
-
-
 @celery_app.task(soft_time_limit=10000)
 def import_candidate_urls_from_api(server_name="test", collection_ids=[]):
     TEMP_FOLDER_NAME = "temp"
@@ -160,31 +78,26 @@ def import_candidate_urls_from_api(server_name="test", collection_ids=[]):
         data_to_import = _get_data_to_import(server_name=server_name, collection=collection)
         print(f"Got {len(data_to_import)} records for {collection.config_folder}")
 
-        print("Clearing DumpUrl model...")
-        DumpUrl.objects.filter(collection=collection).delete()
-
         print("Dumping django fixture to file")
         json.dump(data_to_import, open(urls_file, "w"))
 
-        print("Loading data into Url model using loaddata...")
-        management.call_command("loaddata", urls_file)
+        print("Deleting existing candidate URLs")
+        # this sometimes takes a while
+        collection.candidate_urls.all().delete()
 
-        # TODO: Bishwas wrote this but it is does not work.
-        # print("Creating DumpUrl entries...")
-        # populate_dump_urls(collection)
+        print("Loading fixture; this may take a while")
+        # subprocess.call(f'python manage.py loaddata "{urls_file}"', shell=True)
+        management.call_command(loaddata.Command(), urls_file)
 
         print("Applying existing patterns; this may take a while")
         collection.apply_all_patterns()
 
-        print("Comparing DumpUrl with CuratedUrl...")
-        _compare_and_populate_delta_urls(collection)
-
-        if collection.workflow_status != WorkflowStatusChoices.ENGINEERING_IN_PROGRESS:
+        if collection.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING:
             collection.workflow_status = WorkflowStatusChoices.ENGINEERING_IN_PROGRESS
             collection.save()
 
         # Finally set the status to READY_FOR_CURATION
-        # collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION
+        collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION
         collection.save()
 
     print("Deleting temp files")