implement pattern specificity across entire class

Author: CarsonDavis

sde_collections/models/delta_patterns.py

@@ -40,6 +40,38 @@ class MatchPatternTypeChoices(models.IntegerChoices):
         related_name="%(class)ss",  # Makes curated_url.deltaincludepatterns.all()
     )
 
+    def get_url_match_count(self):
+        """
+        Get the number of unique URLs this pattern matches across both delta and curated URLs.
+        """
+        delta_urls = set(self.get_matching_delta_urls().values_list("url", flat=True))
+        curated_urls = set(self.get_matching_curated_urls().values_list("url", flat=True))
+        return len(delta_urls.union(curated_urls))
+
+    def is_most_distinctive_pattern(self, url) -> bool:
+        """
+        Determine if this pattern should apply to a URL by checking if it matches
+        the smallest number of URLs among all patterns that match this URL.
+        Returns True if this pattern should be applied.
+        """
+        my_match_count = self.get_url_match_count()
+
+        # Get patterns from same type that affect this URL
+        pattern_class = self.__class__
+        matching_patterns = (
+            pattern_class.objects.filter(collection=self.collection)
+            .filter(models.Q(delta_urls__url=url.url) | models.Q(curated_urls__url=url.url))
+            .exclude(id=self.id)
+            .distinct()
+        )
+
+        # If any matching pattern has a smaller URL set, don't apply
+        for pattern in matching_patterns:
+            if pattern.get_url_match_count() < my_match_count:
+                return False
+
+        return True
+
     def get_regex_pattern(self) -> str:
         """Convert the match pattern into a proper regex based on pattern type."""
         escaped_pattern = re.escape(self.match_pattern)
@@ -240,6 +272,9 @@ def apply(self) -> None:
 
         # Create DeltaUrls only where field value would change
         for curated_url in previously_unaffected_curated:
+            if not self.is_most_distinctive_pattern(curated_url):
+                continue
+
             if (
                 getattr(curated_url, field) == new_value
                 or DeltaUrl.objects.filter(url=curated_url.url, collection=self.collection).exists()
@@ -257,8 +292,13 @@ def apply(self) -> None:
 
             DeltaUrl.objects.create(**fields)
 
-        # Update all matching DeltaUrls with the new field value
-        self.get_matching_delta_urls().update(**{field: new_value})
+        # Update all matching DeltaUrls with the new field value if this is the most distinctive pattern
+        for delta_url in self.get_matching_delta_urls():
+            if self.is_most_distinctive_pattern(delta_url):
+                setattr(delta_url, field, new_value)
+                delta_url.save()
+
+        # Update pattern relationships
         self.update_affected_delta_urls_list()
 
     def unapply(self) -> None:
@@ -388,35 +428,6 @@ def generate_title_for_url(self, url_obj) -> tuple[str, str | None]:
         except (ValueError, ValidationError) as e:
             return None, str(e)
 
-    def get_url_match_count(self):
-        """
-        Get the number of unique URLs this pattern matches across both delta and curated URLs.
-        """
-        delta_urls = set(self.get_matching_delta_urls().values_list("url", flat=True))
-        curated_urls = set(self.get_matching_curated_urls().values_list("url", flat=True))
-        return len(delta_urls.union(curated_urls))
-
-    def is_most_distinctive_pattern(self, url) -> bool:
-        """
-        Determine if this pattern should apply to a URL by checking if it matches
-        the smallest number of URLs among all patterns that match this URL.
-        Returns True if this pattern should be applied.
-        """
-        my_match_count = self.get_url_match_count()
-
-        # Get all patterns that match this URL based on match_pattern regex
-        matching_patterns = DeltaTitlePattern.objects.filter(collection=self.collection).exclude(
-            id=self.id
-        )  # Exclude self to avoid duplicate counting
-
-        # Filter to only patterns that would match this URL and get their counts
-        for pattern in matching_patterns:
-            if re.match(pattern.get_regex_pattern(), url.url):
-                if pattern.get_url_match_count() < my_match_count:
-                    return False
-
-        return True
-
     def apply(self) -> None:
         """
         Apply the title pattern to matching URLs:
@@ -491,7 +502,6 @@ def apply(self) -> None:
 
         # Update pattern relationships
         self.update_affected_delta_urls_list()
-        self.update_affected_curated_urls_list()
 
     def unapply(self) -> None:
         """

sde_collections/tests/test_delta_patterns.py

@@ -2,12 +2,12 @@
 
 import pytest
 
-from sde_collections.models.delta_patterns import DeltaExcludePattern, DeltaTitlePattern
-from sde_collections.models.delta_url import (
-    CuratedUrl,
+from sde_collections.models.delta_patterns import (
+    DeltaExcludePattern,
     DeltaResolvedTitleError,
-    DeltaUrl,
+    DeltaTitlePattern,
 )
+from sde_collections.models.delta_url import CuratedUrl, DeltaUrl
 from sde_collections.tests.factories import (
     CollectionFactory,
     CuratedUrlFactory,
@@ -101,7 +101,6 @@ def test_apply_generates_delta_url_if_title_differs(self):
             collection=collection,
             url="https://example.com/page",
             scraped_title="Sample Title",
-            generated_title="Old Title - Processed",
         )
 
         # Step 2: Create a `DeltaTitlePattern` with a new title pattern
@@ -112,9 +111,6 @@ def test_apply_generates_delta_url_if_title_differs(self):
             title_pattern="{title} - Processed New",
         )
 
-        # Apply the pattern
-        pattern.apply()
-
         # Step 3: A new DeltaUrl should be created with the updated `generated_title`
         delta_url = DeltaUrl.objects.get(url=curated_url.url)
         expected_generated_title = resolve_title(
@@ -217,7 +213,7 @@ def test_unapply_removes_pattern_relationships(self):
         curated_url = CuratedUrlFactory(
             collection=collection, url="https://example.com/page", scraped_title="Sample Title"
         )
-        delta_url = DeltaUrlFactory(collection=collection, url="https://example.com/page", scraped_title="Sample Title")
+        delta_url = DeltaUrlFactory(collection=collection, url="https://example.com/page", scraped_title="New Title")
 
         # Create and apply a `DeltaTitlePattern`
         pattern = DeltaTitlePattern.objects.create(
@@ -231,7 +227,8 @@ def test_unapply_removes_pattern_relationships(self):
 
         # Ensure relationships are set
         assert pattern.delta_urls.filter(pk=delta_url.pk).exists()
-        assert pattern.curated_urls.filter(pk=curated_url.pk).exists()
+        # this actually shouldn't match until after promotion
+        assert not pattern.curated_urls.filter(pk=curated_url.pk).exists()
 
         # Unapply the pattern
         pattern.unapply()

…collections/tests/test_title_patterns.py …ctions/tests/test_pattern_specificity.pysde_collections/tests/test_title_patterns.py renamed to sde_collections/tests/test_pattern_specificity.py sde_collections/tests/test_title_patterns.py renamed to sde_collections/tests/test_pattern_specificity.py

@@ -1,8 +1,13 @@
-# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_title_patterns.py
+# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_pattern_specificity.py
 
 import pytest
 
-from sde_collections.models.delta_patterns import DeltaResolvedTitle, DeltaTitlePattern
+from sde_collections.models.collection_choice_fields import DocumentTypes
+from sde_collections.models.delta_patterns import (
+    DeltaDocumentTypePattern,
+    DeltaResolvedTitle,
+    DeltaTitlePattern,
+)
 from sde_collections.models.delta_url import DeltaUrl
 from sde_collections.tests.factories import CollectionFactory, DeltaUrlFactory
 
@@ -77,3 +82,77 @@ def test_title_pattern_multiple_resolved_titles_extended():
     assert url2 in general_pattern.delta_urls.all()
     assert url3 in general_pattern.delta_urls.all()
     assert url3 in specific_pattern.delta_urls.all()
+
+
+@pytest.mark.django_db
+def test_field_modifying_pattern_layered_specificity():
+    """Test overlapping patterns with different levels of specificity."""
+    collection = CollectionFactory()
+
+    # Create URLs in a hierarchy that allows for overlapping pattern matches
+    deep_tool = DeltaUrlFactory(
+        collection=collection,
+        url="https://example.com/tools/analysis/v2/processor.py",
+        document_type=DocumentTypes.DOCUMENTATION,  # Starting as documentation
+    )
+    mid_tool = DeltaUrlFactory(
+        collection=collection,
+        url="https://example.com/tools/analysis/helper.py",
+        document_type=DocumentTypes.DOCUMENTATION,  # Starting as documentation
+    )
+    top_tool = DeltaUrlFactory(
+        collection=collection,
+        url="https://example.com/tools/simple.py",
+        document_type=DocumentTypes.DOCUMENTATION,  # Starting as documentation
+    )
+
+    # Create patterns with overlapping matches
+    broad_pattern = DeltaDocumentTypePattern.objects.create(
+        collection=collection,
+        match_pattern="*/tools/*.py",  # Matches all 3 URLs
+        document_type=DocumentTypes.SOFTWARETOOLS,
+        match_pattern_type=2,
+    )
+
+    mid_pattern = DeltaDocumentTypePattern.objects.create(
+        collection=collection,
+        match_pattern="*/tools/analysis/*.py",  # Matches 2 URLs (mid and deep)
+        document_type=DocumentTypes.DATA,  # Different type to clearly show which pattern won
+        match_pattern_type=2,
+    )
+
+    specific_pattern = DeltaDocumentTypePattern.objects.create(
+        collection=collection,
+        match_pattern="*/analysis/v2/*.py",  # Matches only 1 URL (deep)
+        document_type=DocumentTypes.DOCUMENTATION,  # Different type to clearly show which pattern won
+        match_pattern_type=2,
+    )
+
+    # Verify URL match counts
+    assert broad_pattern.get_url_match_count() == 3
+    assert mid_pattern.get_url_match_count() == 2
+    assert specific_pattern.get_url_match_count() == 1
+
+    # Verify patterns were applied correctly based on specificity
+    deep_tool.refresh_from_db()
+    mid_tool.refresh_from_db()
+    top_tool.refresh_from_db()
+
+    # The most specific pattern (1 match) should win for the deep URL
+    assert deep_tool.document_type == DocumentTypes.DOCUMENTATION, "Deep URL should use most specific pattern"
+
+    # The mid-level pattern (2 matches) should win for the middle URL
+    assert mid_tool.document_type == DocumentTypes.DATA, "Mid URL should use mid-level pattern"
+
+    # The broad pattern (3 matches) should only affect the top URL
+    assert top_tool.document_type == DocumentTypes.SOFTWARETOOLS, "Top URL should use broad pattern"
+
+    # Verify the relationships are tracked correctly
+    assert deep_tool.pk in specific_pattern.delta_urls.values_list("pk", flat=True)
+    assert deep_tool.pk in mid_pattern.delta_urls.values_list("pk", flat=True)
+    assert deep_tool.pk in broad_pattern.delta_urls.values_list("pk", flat=True)
+
+    assert mid_tool.pk in mid_pattern.delta_urls.values_list("pk", flat=True)
+    assert mid_tool.pk in broad_pattern.delta_urls.values_list("pk", flat=True)
+
+    assert top_tool.pk in broad_pattern.delta_urls.values_list("pk", flat=True)