update pattern application to not create excess deltaurls

Author: CarsonDavis

sde_collections/models/delta_patterns.py

@@ -40,7 +40,10 @@ class MatchPatternTypeChoices(models.IntegerChoices):
     )
 
     def matched_urls(self):
-        """Find all URLs matching the pattern."""
+        """
+        Find all URLs matching the pattern.
+        This does not update pattern.delta_urls or pattern.curated_urls.
+        """
         DeltaUrl = apps.get_model("sde_collections", "DeltaUrl")
         CuratedUrl = apps.get_model("sde_collections", "CuratedUrl")
 
@@ -92,6 +95,10 @@ def apply(self, fields_to_copy=None, update_fields=None):
 
         # Step 1: Generate or update DeltaUrls for each matching CuratedUrl
         for curated_url in matched_urls["matching_curated_urls"]:
+            # Check if the curated_url is already linked to this pattern
+            if self.curated_urls.filter(pk=curated_url.pk).exists():
+                # Skip creating a DeltaUrl if the curated_url is already associated with this pattern
+                continue
             self.generate_delta_url(curated_url, fields_to_copy)
 
         # Step 2: Apply updates to fields on matching DeltaUrls

sde_collections/tests/test_delta_patterns.py

@@ -18,6 +18,7 @@
     CollectionFactory,
     CuratedUrlFactory,
     DeltaUrlFactory,
+    DumpUrlFactory,
 )
 from sde_collections.utils.title_resolver import resolve_title
 
@@ -84,6 +85,72 @@ def test_generate_delta_url_creation_and_update(self):
         delta_url.refresh_from_db()
         assert delta_url.scraped_title == original_delta_title
 
+    def test_apply_creates_delta_url_if_curated_url_does_not_exist(self):
+        """
+        Ensures that the `apply` logic creates a new `DeltaUrl` if a matching `CuratedUrl` does not exist.
+        """
+        collection = CollectionFactory()
+        delta_url = DeltaUrlFactory(
+            collection=collection, url="https://example.com/page", scraped_title="Original Title"
+        )
+
+        # Create a pattern matching the URL
+        pattern = DeltaIncludePattern.objects.create(
+            collection=collection, match_pattern="https://example.com/*", match_pattern_type=2
+        )
+
+        # Apply the pattern
+        pattern.apply()
+
+        # Verify that a DeltaUrl is created
+        assert DeltaUrl.objects.filter(url=delta_url.url).exists()
+
+    def test_apply_skips_delta_url_creation_if_curated_url_exists(self):
+        """
+        Ensures that the `apply` logic does not create a new `DeltaUrl` if a matching `CuratedUrl` already exists.
+        """
+        collection = CollectionFactory()
+        delta_url = DeltaUrlFactory(
+            collection=collection, url="https://example.com/page", scraped_title="Original Title"
+        )
+
+        # Create a pattern matching the URL
+        pattern = DeltaIncludePattern.objects.create(
+            collection=collection, match_pattern="https://example.com/*", match_pattern_type=2
+        )
+
+        # Promote the DeltaUrl to a CuratedUrl
+        collection.promote_to_curated()
+        curated_url = CuratedUrl.objects.get(url=delta_url.url)
+
+        # ReApply the pattern
+        pattern.apply()
+
+        # Verify that no DeltaUrl is created after the CuratedUrl exists
+        assert not DeltaUrl.objects.filter(url=curated_url.url).exists()
+
+    def test_apply_creates_delta_url_if_no_curated_url_exists(self):
+        """
+        Ensures that if no `CuratedUrl` exists for a given pattern, a new `DeltaUrl` is created.
+        """
+        collection = CollectionFactory()
+        dump_url = DumpUrlFactory(collection=collection, url="https://example.com/page", scraped_title="New Title")
+
+        # Migrate DumpUrl to DeltaUrl
+        collection.migrate_dump_to_delta()
+
+        # Create a pattern matching the URL
+        pattern = DeltaIncludePattern.objects.create(
+            collection=collection, match_pattern="https://example.com/*", match_pattern_type=2
+        )
+
+        # Apply the pattern
+        pattern.apply()
+
+        # A `DeltaUrl` should now exist
+        delta_url = DeltaUrl.objects.get(url=dump_url.url)
+        assert delta_url.scraped_title == dump_url.scraped_title
+
     def test_apply_and_unapply_pattern(self):
         # if we make a new exclude pattern and it affects an old url
         # that wasn't previously affected, what should happen?
@@ -258,6 +325,30 @@ def test_unapply_removes_pattern_relationships(self):
         assert not pattern.delta_urls.filter(pk=delta_url.pk).exists()
         assert not pattern.curated_urls.filter(pk=curated_url.pk).exists()
 
+    # TODO: work on this test logic
+    # def test_pattern_reapplication_does_not_duplicate_delta_urls(self):
+    #     """
+    #     Ensures that reapplying a pattern does not create duplicate `DeltaUrls` or affect existing `CuratedUrls`.
+    #     """
+    #     collection = CollectionFactory()
+    #     delta_url = DeltaUrlFactory(collection=collection,
+    #                                 url="https://example.com/page",
+    #                                 scraped_title="Title Before")
+
+    #     # Promote to CuratedUrl
+    #     collection.promote_to_curated()
+    #     curated_url = CuratedUrl.objects.get(url=delta_url.url)
+
+    #     # Apply a pattern
+    #     pattern = DeltaTitlePattern.objects.create(
+    #         collection=collection, match_pattern="https://example.com/*", match_pattern_type=2, title_patte......
+    #     )
+    #     pattern.apply()
+
+    #     # Ensure no new `DeltaUrl` is created after reapplying the pattern
+    #     pattern.apply()
+    #     assert DeltaUrl.objects.filter(url=curated_url.url).count() == 0
+
 
 @pytest.mark.django_db
 class TestDeltaDocumentTypePattern: