Merge pull request #1090 from NASA-IMPACT/1051-backend-model-changes-on-cosmos-to-hold-new-incoming-urls-frontend

1051 backend model changes on cosmos to hold new incoming urls frontend

Author: CarsonDavis

README.md

@@ -82,7 +82,7 @@ $ docker-compose -f local.yml run --rm django python manage.py loaddata sde_coll
 Navigate to the server running prod, then to the project folder. Run the following command to create a backup:
 
 ```bash
-docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20240812.json
+docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20241114.json
 ```
 This will have saved the backup in a folder outside of the docker container. Now you can copy it to your local machine.
 
@@ -208,3 +208,20 @@ Eventually, job creation will be done seamlessly by the webapp. Until then, edit
   - JavaScript: `/sde_indexing_helper/static/js`
   - CSS: `/sde_indexing_helper/static/css`
   - Images: `/sde_indexing_helper/static/images`
+
+
+## Running Long Scripts on the Server
+```shell
+tmux new -s docker_django
+```
+Once you are inside, you can run dmshell.
+
+Later, you can do this to get back in.
+```shell
+tmux attach -t docker_django
+```
+
+To delete the session:
+```shell
+tmux kill-session -t docker_django
+```

compose/production/traefik/traefik.yml

@@ -31,7 +31,7 @@ certificatesResolvers:
 http:
   routers:
     web-secure-router:
-      rule: "Host(`sde-indexing-helper.nasa-impact.net`)"
+      rule: 'Host(`{{ env "TRAEFIK_DOMAIN" }}`)'
       entryPoints:
         - web-secure
       middlewares:
@@ -42,7 +42,7 @@ http:
         certResolver: letsencrypt
 
     flower-secure-router:
-      rule: "Host(`sde-indexing-helper.nasa-impact.net`)"
+      rule: 'Host(`{{ env "TRAEFIK_DOMAIN" }}`)'
       entryPoints:
         - flower
       service: flower

config_generation/xmls/plugin_indexing_template.xml

@@ -268,8 +268,4 @@
         <Name>id</Name>
         <Value>doc.url1</Value>
     </Mapping>
-    <Mapping>
-        <Name>version</Name>
-        <Value>Md5(doc.url1)</Value>
-    </Mapping>
 </Sinequa>

production.yml

@@ -38,6 +38,10 @@ services:
       - django
     volumes:
       - production_traefik:/etc/traefik/acme
+    env_file:
+      # this should contain TRAEFIK_DOMAIN=sde-indexing-helper-staging.nasa-impact.net or
+      # TRAEFIK_DOMAIN=sde-indexing-helper.nasa-impact.net or
+      - ./.envs/.production/.traefik
     ports:
       - "0.0.0.0:80:80"
       - "0.0.0.0:443:443"

requirements/base.txt

@@ -25,8 +25,11 @@ django-cors-headers==4.4.0
 django-filter==24.3
 djangorestframework-datatables==0.7.2
 djangorestframework==3.15.2
+factory-boy==3.3.0
 lxml==4.9.2
 PyGithub==2.2.0
+pytest-django==4.8.0
+pytest==8.0.0
 tqdm==4.66.3
 unidecode==1.3.8
 xmltodict==0.13.0

scripts/analyze_and_remove_url_duplicates.py

@@ -0,0 +1,76 @@
+from collections import defaultdict
+
+from django.db import models
+
+from sde_collections.models.candidate_url import CandidateURL
+from sde_collections.models.collection import Collection
+
+# Get all field names except 'id' and 'collection' (since we're already looping by collection)
+duplicate_fields = [field.name for field in CandidateURL._meta.get_fields() if field.name not in ["id", "collection"]]
+
+
+def analyze_duplicates():
+    """Analyze duplicates and print how many would be deleted in each collection."""
+    deletion_stats = defaultdict(lambda: {"total": 0, "to_delete": 0})
+
+    # Loop through each collection
+    for collection in Collection.objects.all():
+        # Count total URLs for the collection
+        total_urls = CandidateURL.objects.filter(collection=collection).count()
+        deletion_stats[collection.config_folder]["total"] = total_urls
+
+        # Group CandidateURL instances by all fields dynamically
+        duplicates_in_collection = (
+            CandidateURL.objects.filter(collection=collection)
+            .values(*duplicate_fields)
+            .annotate(count=models.Count("id"))
+            .filter(count__gt=1)
+        )
+
+        # Count potential deletions without deleting
+        for entry in duplicates_in_collection:
+            duplicates_count = CandidateURL.objects.filter(
+                collection=collection, **{field: entry[field] for field in duplicate_fields}
+            ).count()
+            deletion_stats[collection.config_folder]["to_delete"] += duplicates_count - 1
+
+    # Print analysis results
+    print("Duplicate analysis completed.")
+    for config_folder, stats in deletion_stats.items():
+        print(f"{config_folder}' has {stats['total']} total URL(s), with {stats['to_delete']} duplicates.")
+
+
+def delete_duplicates():
+    """Delete duplicates based on previously analyzed duplicates."""
+    deletion_stats = defaultdict(int)
+
+    # Loop through each collection
+    for collection in Collection.objects.all():
+        # Group CandidateURL instances by all fields dynamically
+        duplicates_in_collection = (
+            CandidateURL.objects.filter(collection=collection)
+            .values(*duplicate_fields)
+            .annotate(count=models.Count("id"))
+            .filter(count__gt=1)
+        )
+
+        # Delete duplicates and track deletions
+        for entry in duplicates_in_collection:
+            duplicates = CandidateURL.objects.filter(
+                collection=collection, **{field: entry[field] for field in duplicate_fields}
+            )
+
+            # Keep the first instance and delete the rest
+            for candidate in duplicates[1:]:  # Skip the first to retain it
+                candidate.delete()
+                deletion_stats[collection.config_folder] += 1
+
+    # Print deletion results
+    print("Duplicate URL cleanup completed.")
+    for config_folder, deleted_count in deletion_stats.items():
+        print(f"Collection '{config_folder}' had {deleted_count} duplicate URL(s) deleted.")
+
+
+# Usage
+analyze_duplicates()  # First analyze duplicates
+delete_duplicates()  # Then delete duplicates based on analysis

sde_collections/admin.py

@@ -3,26 +3,32 @@
 from django.contrib import admin, messages
 from django.http import HttpResponse
 
+from sde_collections.models.delta_patterns import (
+    DeltaDivisionPattern,
+    DeltaTitlePattern,
+)
+
 from .models.candidate_url import CandidateURL, ResolvedTitle
 from .models.collection import Collection, WorkflowHistory
+from .models.delta_url import CuratedUrl, DeltaResolvedTitle, DeltaUrl, DumpUrl
 from .models.pattern import DivisionPattern, IncludePattern, TitlePattern
-from .tasks import fetch_and_update_full_text, import_candidate_urls_from_api
+from .tasks import fetch_and_replace_full_text, import_candidate_urls_from_api
 
 
-def fetch_and_update_text_for_server(modeladmin, request, queryset, server_name):
+def fetch_and_replace_text_for_server(modeladmin, request, queryset, server_name):
     for collection in queryset:
-        fetch_and_update_full_text.delay(collection.id, server_name)
+        fetch_and_replace_full_text.delay(collection.id, server_name)
     modeladmin.message_user(request, f"Started importing URLs from {server_name.upper()} Server")
 
 
 @admin.action(description="Import candidate URLs from LRM Dev Server with Full Text")
 def fetch_full_text_lrm_dev_action(modeladmin, request, queryset):
-    fetch_and_update_text_for_server(modeladmin, request, queryset, "lrm_dev")
+    fetch_and_replace_text_for_server(modeladmin, request, queryset, "lrm_dev")
 
 
 @admin.action(description="Import candidate URLs from XLI Server with Full Text")
-def fetch_full_text_lis_action(modeladmin, request, queryset):
-    fetch_and_update_text_for_server(modeladmin, request, queryset, "xli")
+def fetch_full_text_xli_action(modeladmin, request, queryset):
+    fetch_and_replace_text_for_server(modeladmin, request, queryset, "xli")
 
 
 @admin.action(description="Generate deployment message")
@@ -234,6 +240,8 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
     list_display = (
         "name",
         "candidate_urls_count",
+        "delta_urls_count",
+        "included_curated_urls_count",
         "config_folder",
         "url",
         "division",
@@ -248,15 +256,8 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
         "export_as_csv",
         "update_config",
         download_candidate_urls_as_csv,
-        import_candidate_urls_test,
-        import_candidate_urls_production,
-        import_candidate_urls_secret_test,
-        import_candidate_urls_secret_production,
-        import_candidate_urls_xli_server,
-        import_candidate_urls_lrm_dev_server,
-        import_candidate_urls_lrm_qa_server,
         fetch_full_text_lrm_dev_action,
-        fetch_full_text_lis_action,
+        fetch_full_text_xli_action,
     ]
     ordering = ("cleaning_order",)
 
@@ -317,9 +318,63 @@ class DivisionPatternAdmin(admin.ModelAdmin):
     search_fields = ("match_pattern", "division")
 
 
+# deltas below
+class DeltaTitlePatternAdmin(admin.ModelAdmin):
+    """Admin View for DeltaTitlePattern"""
+
+    list_display = (
+        "match_pattern",
+        "title_pattern",
+        "collection",
+        "match_pattern_type",
+    )
+    list_filter = (
+        "match_pattern_type",
+        "collection",
+    )
+
+
+class DeltaResolvedTitleAdmin(admin.ModelAdmin):
+    list_display = ["title_pattern", "delta_url", "resolved_title", "created_at"]
+
+
+class DeltaDivisionPatternAdmin(admin.ModelAdmin):
+    list_display = ("collection", "match_pattern", "division")
+    search_fields = ("match_pattern", "division")
+
+
+class DumpUrlAdmin(admin.ModelAdmin):
+    """Admin View for DumpUrl"""
+
+    list_display = ("url", "scraped_title", "collection")
+    list_filter = ("collection",)
+
+
+class DeltaUrlAdmin(admin.ModelAdmin):
+    """Admin View for DeltaUrl"""
+
+    list_display = ("url", "scraped_title", "generated_title", "collection")
+    list_filter = ("collection",)
+
+
+class CuratedUrlAdmin(admin.ModelAdmin):
+    """Admin View for CuratedUrl"""
+
+    list_display = ("url", "scraped_title", "generated_title", "collection")
+    list_filter = ("collection",)
+
+
 admin.site.register(WorkflowHistory, WorkflowHistoryAdmin)
 admin.site.register(CandidateURL, CandidateURLAdmin)
 admin.site.register(TitlePattern, TitlePatternAdmin)
 admin.site.register(IncludePattern)
 admin.site.register(ResolvedTitle, ResolvedTitleAdmin)
 admin.site.register(DivisionPattern, DivisionPatternAdmin)
+
+
+admin.site.register(DeltaTitlePattern, DeltaTitlePatternAdmin)
+admin.site.register(DeltaResolvedTitle, DeltaResolvedTitleAdmin)
+admin.site.register(DeltaDivisionPattern, DeltaDivisionPatternAdmin)
+admin.site.register(DumpUrl, DumpUrlAdmin)
+admin.site.register(DeltaUrl, DeltaUrlAdmin)
+admin.site.register(CuratedUrl, CuratedUrlAdmin)