fix tests and refactor related_names for patterns

Author: CarsonDavis

sde_collections/migrations/0068_alter_deltadivisionpattern_collection_and_more.py

@@ -0,0 +1,124 @@
+# Generated by Django 4.2.9 on 2024-11-24 19:39
+
+from django.db import migrations, models
+import django.db.models.deletion
+import sde_collections.models.delta_patterns
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0067_alter_deltadivisionpattern_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="deltadivisionpattern",
+            name="collection",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="%(class)ss",
+                related_query_name="%(class)ss",
+                to="sde_collections.collection",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="deltadivisionpattern",
+            name="curated_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.curatedurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltadivisionpattern",
+            name="delta_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.deltaurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltadocumenttypepattern",
+            name="collection",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="%(class)ss",
+                related_query_name="%(class)ss",
+                to="sde_collections.collection",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="deltadocumenttypepattern",
+            name="curated_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.curatedurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltadocumenttypepattern",
+            name="delta_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.deltaurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltaexcludepattern",
+            name="collection",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="%(class)ss",
+                related_query_name="%(class)ss",
+                to="sde_collections.collection",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="deltaexcludepattern",
+            name="curated_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.curatedurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltaexcludepattern",
+            name="delta_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.deltaurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltaincludepattern",
+            name="collection",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="%(class)ss",
+                related_query_name="%(class)ss",
+                to="sde_collections.collection",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="deltaincludepattern",
+            name="curated_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.curatedurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltaincludepattern",
+            name="delta_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.deltaurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltatitlepattern",
+            name="collection",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="%(class)ss",
+                related_query_name="%(class)ss",
+                to="sde_collections.collection",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="deltatitlepattern",
+            name="curated_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.curatedurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltatitlepattern",
+            name="delta_urls",
+            field=models.ManyToManyField(related_name="%(class)ss", to="sde_collections.deltaurl"),
+        ),
+        migrations.AlterField(
+            model_name="deltatitlepattern",
+            name="title_pattern",
+            field=models.CharField(
+                help_text="Pattern for the new title. Can be an exact replacement string or sinequa-valid code",
+                validators=[sde_collections.models.delta_patterns.validate_title_pattern],
+                verbose_name="Title Pattern",
+            ),
+        ),
+    ]

sde_collections/models/collection.py

@@ -118,7 +118,12 @@ def refresh_url_lists_for_all_patterns(self):
                 pattern.update_affected_curated_urls_list()
 
     def migrate_dump_to_delta(self):
-        """Main function to handle migration from DumpUrls to DeltaUrls with specific rules."""
+        """
+        Migrates data from DumpUrls to DeltaUrls, preserving all fields.
+        Creates DeltaUrls that reflect:
+        1. Changes from DumpUrls vs CuratedUrls
+        2. Missing URLs in DumpUrls that exist in CuratedUrls (marked for deletion)
+        """
         # Step 1: Clear existing DeltaUrls for this collection
         self.clear_delta_urls()
 
@@ -146,27 +151,31 @@ def migrate_dump_to_delta(self):
         # Step 5: Clear DumpUrls after migration is complete
         self.clear_dump_urls()
 
-        # Step 6: Reapply patterns to DeltaUrls
-        self.refresh_url_lists_for_all_patterns()
+        # Step 6: Apply all patterns to DeltaUrls
+        # self.refresh_url_lists_for_all_patterns() # TODO: I'm pretty confident we shouldn't be running this
+        self.apply_all_patterns()
 
     def create_or_update_delta_url(self, url_instance, to_delete=False):
         """
         Creates or updates a DeltaUrl entry based on the given DumpUrl or CuratedUrl object.
-        If to_delete is True, only sets the to_delete flag and url.
+        Always copies all fields, even for deletion cases.
+
+        Args:
+            url_instance: DumpUrl or CuratedUrl instance to copy from
+            to_delete: Whether to mark the resulting DeltaUrl for deletion
         """
-        if to_delete:
-            # Only set the URL and to_delete flag
-            DeltaUrl.objects.update_or_create(collection=self, url=url_instance.url, defaults={"to_delete": True})
-        else:
-            # Automatically move over all fields from url_instance
-            fields_to_copy = {
-                field.name: getattr(url_instance, field.name)
-                for field in DumpUrl._meta.fields  # Assumes same fields for CuratedUrl via inheritance
-                if field.name not in ["id", "collection", "url"]
-            }
-            fields_to_copy["to_delete"] = False  # Ensure to_delete flag is False
+        # Get all copyable fields from the source instance
+        fields_to_copy = {
+            field.name: getattr(url_instance, field.name)
+            for field in url_instance._meta.fields
+            if field.name not in ["id", "collection"]
+        }
 
-            DeltaUrl.objects.update_or_create(collection=self, url=url_instance.url, defaults=fields_to_copy)
+        # Set deletion status
+        fields_to_copy["to_delete"] = to_delete
+
+        # Update or create the DeltaUrl
+        DeltaUrl.objects.update_or_create(collection=self, url=url_instance.url, defaults=fields_to_copy)
 
     def promote_to_curated(self):
         """
@@ -600,15 +609,32 @@ def sync_with_production_webapp(self) -> None:
 
         self.save()
 
-    def apply_all_patterns(self) -> None:
-        """Apply all the patterns."""
-        for pattern in self.excludepattern.all():
+    def apply_all_patterns(self):
+        """Apply all the patterns with debug information."""
+        print("\nApplying patterns:")
+
+        for pattern in self.deltaexcludepatterns.all():
+            print(f"\nApplying exclude pattern: {pattern.match_pattern}")
             pattern.apply()
-        for pattern in self.includepattern.all():
+
+        for pattern in self.deltaincludepatterns.all():
+            print(f"\nApplying include pattern: {pattern.match_pattern}")
             pattern.apply()
-        for pattern in self.titlepattern.all():
+
+        for pattern in self.deltatitlepatterns.all():
+            print(f"\nApplying title pattern: {pattern.match_pattern}")
             pattern.apply()
-        for pattern in self.documenttypepattern.all():
+
+        for pattern in self.deltadocumenttypepatterns.all():
+            print(f"\nApplying doctype pattern: {pattern.match_pattern}")
+            matching_urls = pattern.get_matching_delta_urls()
+            print(f"Matching URLs: {matching_urls.count()}")
+            pattern.apply()
+
+        for pattern in self.deltadivisionpatterns.all():
+            print(f"\nApplying division pattern: {pattern.match_pattern}")
+            matching_urls = pattern.get_matching_delta_urls()
+            print(f"Matching URLs: {matching_urls.count()}")
             pattern.apply()
 
     def save(self, *args, **kwargs):

sde_collections/models/delta_patterns.py

@@ -24,7 +24,7 @@ class MatchPatternTypeChoices(models.IntegerChoices):
     collection = models.ForeignKey(
         "Collection",
         on_delete=models.CASCADE,
-        related_name="%(class)s",
+        related_name="%(class)ss",  # Makes collection.deltaincludepatterns.all()
         related_query_name="%(class)ss",
     )
     match_pattern = models.CharField(
@@ -33,11 +33,11 @@ class MatchPatternTypeChoices(models.IntegerChoices):
     match_pattern_type = models.IntegerField(choices=MatchPatternTypeChoices.choices, default=1)
     delta_urls = models.ManyToManyField(
         "DeltaUrl",
-        related_name="%(class)s_delta_urls",
+        related_name="%(class)ss",  # Makes delta_url.deltaincludepatterns.all()
     )
     curated_urls = models.ManyToManyField(
         "CuratedUrl",
-        related_name="%(class)s_curated_urls",
+        related_name="%(class)ss",  # Makes curated_url.deltaincludepatterns.all()
     )
 
     def get_regex_pattern(self) -> str:

sde_collections/tests/factories.py

@@ -66,25 +66,25 @@ class Meta:
     # division = 1
 
 
-class CuratedUrlFactory(factory.django.DjangoModelFactory):
+class DeltaUrlFactory(factory.django.DjangoModelFactory):
     class Meta:
-        model = CuratedUrl
+        model = DeltaUrl
 
     collection = factory.SubFactory(CollectionFactory)
     url = factory.Faker("url")
     scraped_title = factory.Faker("sentence")
-    scraped_text = factory.Faker("paragraph")
-    generated_title = factory.Faker("sentence")
-    visited = factory.Faker("boolean")
-    document_type = 1
-    division = 1
+    to_delete = False
 
 
-class DeltaUrlFactory(factory.django.DjangoModelFactory):
+class CuratedUrlFactory(factory.django.DjangoModelFactory):
     class Meta:
-        model = DeltaUrl
+        model = CuratedUrl
 
     collection = factory.SubFactory(CollectionFactory)
     url = factory.Faker("url")
     scraped_title = factory.Faker("sentence")
-    to_delete = False
+    scraped_text = factory.Faker("paragraph")
+    generated_title = factory.Faker("sentence")
+    visited = factory.Faker("boolean")
+    document_type = 1
+    division = 1