updated patterns to refresh url lists

Author: CarsonDavis

sde_collections/models/collection.py

@@ -3,6 +3,7 @@
 
 import requests
 from django.contrib.auth import get_user_model
+from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.db.models.signals import post_save
 from django.dispatch import receiver
@@ -93,6 +94,28 @@ def clear_dump_urls(self):
         """Clears all DumpUrls for this collection."""
         DumpUrl.objects.filter(collection=self).delete()
 
+    def refresh_url_lists_for_all_patterns(self):
+        """
+        Updates pattern relations for all patterns associated with this collection.
+        """
+        # List of pattern models to update
+        pattern_models = [
+            "DeltaExcludePattern",
+            "DeltaIncludePattern",
+            "DeltaTitlePattern",
+            "DeltaDocumentTypePattern",
+            "DeltaDivisionPattern",
+        ]
+
+        # Loop through each model and update its relations
+        for model_name in pattern_models:
+            # Get the model dynamically
+            model = ContentType.objects.get(app_label="sde_collections", model=model_name.lower()).model_class()
+
+            # Filter patterns for the current collection and update relations
+            for pattern in model.objects.filter(collection=self):
+                pattern.refresh_url_lists()
+
     def migrate_dump_to_delta(self):
         """Main function to handle migration from DumpUrls to DeltaUrls with specific rules."""
         # Step 1: Clear existing DeltaUrls for this collection
@@ -122,6 +145,9 @@ def migrate_dump_to_delta(self):
         # Step 5: Clear DumpUrls after migration is complete
         self.clear_dump_urls()
 
+        # Step 6: Reapply patterns to DeltaUrls
+        self.refresh_url_lists_for_all_patterns()
+
     def create_or_update_delta_url(self, url_instance, to_delete=False):
         """
         Creates or updates a DeltaUrl entry based on the given DumpUrl or CuratedUrl object.
@@ -172,7 +198,6 @@ def promote_to_curated(self):
                         updated_fields[field_name] = delta_value
 
                 if updated_fields:
-                    # Use update to modify fields directly in the database
                     CuratedUrl.objects.filter(pk=curated.pk).update(**updated_fields)
             else:
                 # If no matching CuratedUrl, create a new one using all non-null and non-empty fields
@@ -186,6 +211,9 @@ def promote_to_curated(self):
         # Step 3: Clear all DeltaUrls for this collection since they've been promoted
         DeltaUrl.objects.filter(collection=self).delete()
 
+        # Step 4: Reapply patterns to DeltaUrls
+        self.refresh_url_lists_for_all_patterns()
+
     def add_to_public_query(self):
         """Add the collection to the public query."""
         if self.workflow_status not in [

sde_collections/models/delta_patterns.py

@@ -41,7 +41,6 @@ class MatchPatternTypeChoices(models.IntegerChoices):
 
     def matched_urls(self):
         """Find all URLs matching the pattern."""
-        # Dynamically get the DeltaUrl model to avoid circular imports
         DeltaUrl = apps.get_model("sde_collections", "DeltaUrl")
         CuratedUrl = apps.get_model("sde_collections", "CuratedUrl")
 
@@ -62,6 +61,12 @@ def matched_urls(self):
             "matching_curated_urls": matching_curated_urls,
         }
 
+    def refresh_url_lists(self):
+        """Update the delta_urls and curated_urls ManyToMany relationships."""
+        matched_urls = self.matched_urls()
+        self.delta_urls.set(matched_urls["matching_delta_urls"])
+        self.curated_urls.set(matched_urls["matching_curated_urls"])
+
     def generate_delta_url(self, curated_url, fields_to_copy=None):
         """
         Generates or updates a DeltaUrl based on a CuratedUrl.
@@ -93,9 +98,8 @@ def apply(self, fields_to_copy=None, update_fields=None):
         if update_fields:
             matched_urls["matching_delta_urls"].update(**update_fields)
 
-        # Step 3: Populate ManyToMany relationships for DeltaUrls and CuratedUrls
-        self.delta_urls.add(*matched_urls["matching_delta_urls"])
-        self.curated_urls.add(*matched_urls["matching_curated_urls"])
+        # Update ManyToMany relationships
+        self.refresh_url_lists()
 
     def unapply(self):
         """Default unapply behavior."""
@@ -182,8 +186,7 @@ def apply(self) -> None:
             self.create_delta_if_title_differs(curated_url, DeltaResolvedTitle, DeltaResolvedTitleError)
 
         # Step 3: Update ManyToMany relationships for DeltaUrls and CuratedUrls
-        self.delta_urls.add(*matched_urls["matching_delta_urls"])
-        self.curated_urls.add(*matched_urls["matching_curated_urls"])
+        self.refresh_url_lists()
 
     def create_delta_if_title_differs(self, curated_url, DeltaResolvedTitle, DeltaResolvedTitleError):
         """

sde_collections/tests/test_migrate_dump.py

@@ -3,6 +3,10 @@
 
 import pytest
 
+from sde_collections.models.delta_patterns import (
+    DeltaExcludePattern,
+    DeltaIncludePattern,
+)
 from sde_collections.models.delta_url import DeltaUrl, DumpUrl
 from sde_collections.tests.factories import (
     CollectionFactory,
@@ -60,7 +64,7 @@ def test_new_url_in_dump_only(self):
     def test_url_in_both_with_different_field(self):
         collection = CollectionFactory()
         dump_url = DumpUrlFactory(collection=collection, scraped_title="New Title")
-        curated_url = CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Old Title")  # noqa
+        CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Old Title")
         collection.migrate_dump_to_delta()
         delta = DeltaUrl.objects.get(url=dump_url.url)
         assert delta.to_delete is False
@@ -77,7 +81,7 @@ def test_url_in_curated_only(self):
     def test_identical_url_in_both(self):
         collection = CollectionFactory()
         dump_url = DumpUrlFactory(collection=collection, scraped_title="Same Title")
-        curated_url = CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Same Title")  # noqa
+        CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Same Title")
         collection.migrate_dump_to_delta()
         assert not DeltaUrl.objects.filter(url=dump_url.url).exists()
 
@@ -200,7 +204,7 @@ def test_full_migration_deleted_url(self):
 def test_empty_delta_comparison_fields():
     collection = CollectionFactory()
     dump_url = DumpUrlFactory(collection=collection, scraped_title="Same Title")
-    curated_url = CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Same Title")  # noqa
+    CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Same Title")  # noqa
 
     global DELTA_COMPARISON_FIELDS
     original_fields = DELTA_COMPARISON_FIELDS
@@ -218,11 +222,88 @@ def test_empty_delta_comparison_fields():
 def test_partial_data_in_curated_urls():
     collection = CollectionFactory()
     dump_url = DumpUrlFactory(collection=collection, scraped_title="Title Exists")
-    curated_url = CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="")  # noqa
+    CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="")  # noqa
 
     collection.migrate_dump_to_delta()
 
     # Since `scraped_title` differs (None vs "Title Exists"), it should create a DeltaUrl
     delta = DeltaUrl.objects.get(url=dump_url.url)
     assert delta.scraped_title == "Title Exists"
     assert delta.to_delete is False
+
+
+@pytest.mark.django_db
+def test_patterns_applied_after_migration():
+    collection = CollectionFactory()
+
+    # Add DumpUrls to migrate
+    DumpUrlFactory(collection=collection, url="https://exclude.com")
+    DumpUrlFactory(collection=collection, url="https://include.com")
+    DumpUrlFactory(collection=collection, url="https://neutral.com")
+
+    # Create exclude and include patterns
+    exclude_pattern = DeltaExcludePattern.objects.create(
+        collection=collection, match_pattern_type=2, match_pattern="exclude.*"
+    )
+    include_pattern = DeltaIncludePattern.objects.create(
+        collection=collection, match_pattern_type=2, match_pattern="include.*"
+    )
+
+    # Perform the migration
+    collection.migrate_dump_to_delta()
+
+    # Check that the patterns were applied
+    exclude_pattern.refresh_from_db()
+    include_pattern.refresh_from_db()
+
+    # Verify exclude pattern relationship
+    assert exclude_pattern.delta_urls.filter(
+        url="https://exclude.com"
+    ).exists(), "Exclude pattern not applied to DeltaUrls."
+
+    # Verify include pattern relationship
+    assert include_pattern.delta_urls.filter(
+        url="https://include.com"
+    ).exists(), "Include pattern not applied to DeltaUrls."
+
+    # Ensure neutral URL is unaffected
+    assert not exclude_pattern.delta_urls.filter(
+        url="https://neutral.com"
+    ).exists(), "Exclude pattern incorrectly applied."
+    assert not include_pattern.delta_urls.filter(
+        url="https://neutral.com"
+    ).exists(), "Include pattern incorrectly applied."
+
+
+@pytest.mark.django_db
+def test_full_migration_with_patterns():
+    collection = CollectionFactory()
+
+    # Set up DumpUrls and CuratedUrls
+    DumpUrlFactory(collection=collection, url="https://new.com")
+    DumpUrlFactory(collection=collection, url="https://update.com", scraped_title="Updated Title")
+    CuratedUrlFactory(collection=collection, url="https://update.com", scraped_title="Old Title")
+    CuratedUrlFactory(collection=collection, url="https://delete.com")
+
+    # Create patterns
+    exclude_pattern = DeltaExcludePattern.objects.create(
+        collection=collection, match_pattern_type=2, match_pattern="delete.*"
+    )
+    include_pattern = DeltaIncludePattern.objects.create(
+        collection=collection, match_pattern_type=2, match_pattern="update.*"
+    )
+
+    # Perform migration
+    collection.migrate_dump_to_delta()
+
+    # Check DeltaUrls
+    assert DeltaUrl.objects.filter(url="https://new.com", to_delete=False).exists()
+    assert DeltaUrl.objects.filter(url="https://update.com", to_delete=False, scraped_title="Updated Title").exists()
+    assert DeltaUrl.objects.filter(url="https://delete.com", to_delete=True).exists()
+
+    # Check patterns
+    exclude_pattern.refresh_from_db()
+    include_pattern.refresh_from_db()
+
+    assert exclude_pattern.delta_urls.filter(url="https://delete.com").exists(), "Exclude pattern not applied."
+    assert include_pattern.delta_urls.filter(url="https://update.com").exists(), "Include pattern not applied."

sde_collections/tests/test_promote_collection.py

@@ -2,6 +2,10 @@
 
 import pytest
 
+from sde_collections.models.delta_patterns import (
+    DeltaExcludePattern,
+    DeltaIncludePattern,
+)
 from sde_collections.models.delta_url import CuratedUrl, DeltaUrl
 from sde_collections.tests.factories import CollectionFactory
 
@@ -77,3 +81,38 @@ def test_promotion_deletes_curated_urls(collection):
     assert not CuratedUrl.objects.filter(url="https://example1.com").exists()
     # Ensure the other CuratedUrl is still present
     assert CuratedUrl.objects.filter(url="https://example2.com").exists()
+
+
+@pytest.mark.django_db
+def test_patterns_reapplied_after_promotion(collection):
+    # Add DeltaUrls matching the patterns
+    DeltaUrl.objects.create(collection=collection, url="https://exclude.com", scraped_title="Exclude This")
+    DeltaUrl.objects.create(collection=collection, url="https://include.com", scraped_title="Include This")
+
+    # Create exclude and include patterns
+    exclude_pattern = DeltaExcludePattern.objects.create(
+        collection=collection, match_pattern_type=2, match_pattern="exclude.*"
+    )
+    include_pattern = DeltaIncludePattern.objects.create(
+        collection=collection, match_pattern_type=2, match_pattern="include.*"
+    )
+
+    # Promote DeltaUrls to CuratedUrls
+    collection.promote_to_curated()
+
+    # Refresh the patterns and check relationships
+    exclude_pattern.refresh_from_db()
+    include_pattern.refresh_from_db()
+
+    # Verify that patterns are reapplied
+    curated_urls = CuratedUrl.objects.filter(collection=collection)
+
+    assert curated_urls.filter(url="https://exclude.com").exists()
+    assert curated_urls.filter(url="https://include.com").exists()
+
+    # Ensure exclude_pattern and include_pattern relationships are populated
+    assert exclude_pattern.curated_urls.filter(url="https://exclude.com").exists()
+    assert include_pattern.curated_urls.filter(url="https://include.com").exists()
+
+    # Verify exclusion status
+    assert curated_urls.filter(url="https://exclude.com", excluded=True).exists()