Merge branch 'dev' into dependabot/pip/django-model-utils-4.5.1

Author: code-geek

sde_collections/admin.py

@@ -5,7 +5,7 @@
 
 from .models.candidate_url import CandidateURL, ResolvedTitle
 from .models.collection import Collection, WorkflowHistory
-from .models.pattern import IncludePattern, TitlePattern
+from .models.pattern import DivisionPattern, IncludePattern, TitlePattern
 from .tasks import import_candidate_urls_from_api
 
 
@@ -191,6 +191,7 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
                     "update_frequency",
                     "source",
                     "turned_on",
+                    "is_multi_division",
                 ),
             },
         ),
@@ -221,9 +222,10 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
         "url",
         "division",
         "new_collection",
+        "is_multi_division",
     )
     readonly_fields = ("config_folder",)
-    list_filter = ("division", "curation_status", "workflow_status", "turned_on")
+    list_filter = ("division", "curation_status", "workflow_status", "turned_on", "is_multi_division")
     search_fields = ("name", "url", "config_folder")
     actions = [
         generate_deployment_message,
@@ -292,8 +294,14 @@ class ResolvedTitleAdmin(admin.ModelAdmin):
     list_display = ["title_pattern", "candidate_url", "resolved_title", "created_at"]
 
 
+class DivisionPatternAdmin(admin.ModelAdmin):
+    list_display = ("collection", "match_pattern", "division")
+    search_fields = ("match_pattern", "division")
+
+
 admin.site.register(WorkflowHistory, WorkflowHistoryAdmin)
 admin.site.register(CandidateURL, CandidateURLAdmin)
 admin.site.register(TitlePattern, TitlePatternAdmin)
 admin.site.register(IncludePattern)
 admin.site.register(ResolvedTitle, ResolvedTitleAdmin)
+admin.site.register(DivisionPattern, DivisionPatternAdmin)

sde_collections/migrations/0058_candidateurl_division_collection_is_multi_division_and_more.py

@@ -0,0 +1,82 @@
+# Generated by Django 4.2.9 on 2024-08-26 02:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0057_alter_collection_workflow_status_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="candidateurl",
+            name="division",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Astrophysics"),
+                    (2, "Biological and Physical Sciences"),
+                    (3, "Earth Science"),
+                    (4, "Heliophysics"),
+                    (5, "Planetary Science"),
+                    (6, "General"),
+                ],
+                null=True,
+            ),
+        ),
+        migrations.AddField(
+            model_name="collection",
+            name="is_multi_division",
+            field=models.BooleanField(default=False, verbose_name="Is Multi-Division?"),
+        ),
+        migrations.CreateModel(
+            name="DivisionPattern",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                (
+                    "match_pattern",
+                    models.CharField(
+                        help_text="This pattern is compared against the URL of all the documents in the collection and matching documents will be returned",
+                        verbose_name="Pattern",
+                    ),
+                ),
+                (
+                    "match_pattern_type",
+                    models.IntegerField(choices=[(1, "Individual URL Pattern"), (2, "Multi-URL Pattern")], default=1),
+                ),
+                (
+                    "division",
+                    models.IntegerField(
+                        choices=[
+                            (1, "Astrophysics"),
+                            (2, "Biological and Physical Sciences"),
+                            (3, "Earth Science"),
+                            (4, "Heliophysics"),
+                            (5, "Planetary Science"),
+                            (6, "General"),
+                        ]
+                    ),
+                ),
+                (
+                    "candidate_urls",
+                    models.ManyToManyField(related_name="%(class)s_urls", to="sde_collections.candidateurl"),
+                ),
+                (
+                    "collection",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="%(class)s",
+                        related_query_name="%(class)ss",
+                        to="sde_collections.collection",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Division Pattern",
+                "verbose_name_plural": "Division Patterns",
+                "unique_together": {("collection", "match_pattern")},
+            },
+        ),
+    ]

sde_collections/models/candidate_url.py

@@ -5,7 +5,7 @@
 from django.db import models
 
 from .collection import Collection
-from .collection_choice_fields import DocumentTypes
+from .collection_choice_fields import Divisions, DocumentTypes
 from .pattern import ExcludePattern, TitlePattern
 
 
@@ -57,6 +57,7 @@ class CandidateURL(models.Model):
     visited = models.BooleanField(default=False)
     objects = CandidateURLManager()
     document_type = models.IntegerField(choices=DocumentTypes.choices, null=True)
+    division = models.IntegerField(choices=Divisions.choices, null=True)
     inferenced_by = models.CharField(
         "Inferenced By",
         default="",

sde_collections/models/collection.py

@@ -46,6 +46,7 @@ class Collection(models.Model):
     document_type = models.IntegerField(choices=DocumentTypes.choices, default=DocumentTypes.DOCUMENTATION)
     tree_root_deprecated = models.CharField("Tree Root", max_length=1024, default="", blank=True)
     delete = models.BooleanField(default=False)
+    is_multi_division = models.BooleanField("Is Multi-Division?", default=False)
 
     # audit columns for production
     audit_hierarchy = models.CharField("Audit Hierarchy", max_length=2048, default="", blank=True)

sde_collections/models/pattern.py

@@ -10,7 +10,7 @@
     parse_title,
     resolve_title,
 )
-from .collection_choice_fields import DocumentTypes
+from .collection_choice_fields import Divisions, DocumentTypes
 
 
 class BaseMatchPattern(models.Model):
@@ -250,6 +250,29 @@ class Meta:
         unique_together = ("collection", "match_pattern")
 
 
+class DivisionPattern(BaseMatchPattern):
+    division = models.IntegerField(choices=Divisions.choices)
+
+    def apply(self) -> None:
+        matched_urls = self.matched_urls()
+        matched_urls.update(division=self.division)
+        candidate_url_ids = list(matched_urls.values_list("id", flat=True))
+        self.candidate_urls.through.objects.bulk_create(
+            objs=[
+                DivisionPattern.candidate_urls.through(candidateurl_id=candidate_url_id, divisionpattern_id=self.id)
+                for candidate_url_id in candidate_url_ids
+            ]
+        )
+
+    def unapply(self) -> None:
+        self.candidate_urls.update(division=None)
+
+    class Meta:
+        verbose_name = "Division Pattern"
+        verbose_name_plural = "Division Patterns"
+        unique_together = ("collection", "match_pattern")
+
+
 # @receiver(post_save, sender=TitlePattern)
 # def post_save_handler(sender, instance, created, **kwargs):
 #     if created:

sde_collections/serializers.py

@@ -2,8 +2,9 @@
 
 from .models.candidate_url import CandidateURL
 from .models.collection import Collection, WorkflowHistory
-from .models.collection_choice_fields import DocumentTypes
+from .models.collection_choice_fields import Divisions, DocumentTypes
 from .models.pattern import (
+    DivisionPattern,
     DocumentTypePattern,
     ExcludePattern,
     IncludePattern,
@@ -56,6 +57,7 @@ class Meta:
 class CandidateURLSerializer(serializers.ModelSerializer):
     excluded = serializers.BooleanField(required=False)
     document_type_display = serializers.CharField(source="get_document_type_display", read_only=True)
+    division_display = serializers.CharField(source="get_division_display", read_only=True)
     url = serializers.CharField(required=False)
     generated_title_id = serializers.SerializerMethodField(read_only=True)
     match_pattern_type = serializers.SerializerMethodField(read_only=True)
@@ -86,6 +88,8 @@ class Meta:
             "candidate_urls_count",
             "document_type",
             "document_type_display",
+            "division",
+            "division_display",
             "visited",
             "test_title",
             "production_title",
@@ -135,7 +139,13 @@ def get_file_extension(self, obj):
         return obj.fileext
 
     def get_tree_root(self, obj):
-        return obj.collection.tree_root
+        if obj.collection.is_multi_division:
+            if obj.division:
+                return f"/{obj.get_division_display()}/{obj.collection.config_folder}"
+            else:
+                return f"/{obj.collection.get_division_display()}/{obj.collection.config_folder}"
+        else:
+            return obj.collection.tree_root
 
 
 class BasePatternSerializer(serializers.ModelSerializer):
@@ -212,3 +222,26 @@ def validate_match_pattern(self, value):
         except DocumentTypePattern.DoesNotExist:
             pass
         return value
+
+
+class DivisionPatternSerializer(BasePatternSerializer, serializers.ModelSerializer):
+    division_display = serializers.CharField(source="get_division_display", read_only=True)
+    division = serializers.ChoiceField(choices=Divisions.choices)
+
+    class Meta:
+        model = DivisionPattern
+        fields = BasePatternSerializer.Meta.fields + (
+            "division",
+            "division_display",
+        )
+
+    def validate_match_pattern(self, value):
+        try:
+            division_pattern = DivisionPattern.objects.get(
+                match_pattern=value,
+                match_pattern_type=DivisionPattern.MatchPatternTypeChoices.INDIVIDUAL_URL,
+            )
+            division_pattern.delete()
+        except DivisionPattern.DoesNotExist:
+            pass
+        return value

sde_collections/urls.py

@@ -13,6 +13,7 @@
 router.register(r"include-patterns", views.IncludePatternViewSet)
 router.register(r"title-patterns", views.TitlePatternViewSet)
 router.register(r"document-type-patterns", views.DocumentTypePatternViewSet)
+router.register(r"division-patterns", views.DivisionPatternViewSet)
 router.register(r"environmental-justice", EnvironmentalJusticeRowViewSet)
 
 app_name = "sde_collections"
@@ -31,6 +32,7 @@
         views.IndexingInstructionsView.as_view(),
         name="indexing_instructions",
     ),
+    path("api/assign-division/<int:pk>/", views.CandidateURLViewSet.as_view({"post": "update_division"})),
     path(
         "delete-required-url/<int:pk>",
         view=views.RequiredUrlsDeleteView.as_view(),

sde_collections/views.py

@@ -28,6 +28,7 @@
     WorkflowStatusChoices,
 )
 from .models.pattern import (
+    DivisionPattern,
     DocumentTypePattern,
     ExcludePattern,
     IncludePattern,
@@ -39,11 +40,11 @@
     CandidateURLSerializer,
     CollectionReadSerializer,
     CollectionSerializer,
+    DivisionPatternSerializer,
     DocumentTypePatternSerializer,
     ExcludePatternSerializer,
     IncludePatternSerializer,
     TitlePatternSerializer,
-    WorkflowHistorySerializer,
 )
 from .tasks import push_to_github_task
 from .utils.health_check import generate_db_github_metadata_differences
@@ -156,15 +157,15 @@ def get_context_data(self, **kwargs):
             timeline_history[history.workflow_status] = history
 
         # Add placeholders for stages with no workflow history
-        for status in WorkflowStatusChoices:
-            if status not in timeline_history:
-                timeline_history[status] = {
-                    "workflow_status": status,
+        for workflow_status in WorkflowStatusChoices:
+            if workflow_status not in timeline_history:
+                timeline_history[workflow_status] = {
+                    "workflow_status": workflow_status,
                     "created_at": None,
-                    "label": WorkflowStatusChoices(status).label,
+                    "label": WorkflowStatusChoices(workflow_status).label,
                 }
 
-        context["timeline_history"] = [timeline_history[status] for status in WorkflowStatusChoices]
+        context["timeline_history"] = [timeline_history[workflow_status] for workflow_status in WorkflowStatusChoices]
         context["required_urls"] = RequiredUrls.objects.filter(collection=self.get_object())
         context["segment"] = "collection-detail"
         context["comments"] = Comments.objects.filter(collection=self.get_object()).order_by("-created_at")
@@ -220,6 +221,7 @@ def get_context_data(self, **kwargs):
         )  # 2=regex patterns
         context["title_patterns"] = self.collection.titlepattern.all()
         context["workflow_status_choices"] = WorkflowStatusChoices
+        context["is_multi_division"] = self.collection.is_multi_division
 
         return context
 
@@ -272,6 +274,15 @@ def get_queryset(self):
                 queryset = self._filter_by_is_excluded(queryset, is_excluded)
         return queryset.order_by("url")
 
+    def update_division(self, request, pk=None):
+        candidate_url = get_object_or_404(CandidateURL, pk=pk)
+        division = request.data.get("division")
+        if division:
+            candidate_url.division = division
+            candidate_url.save()
+            return Response(status=status.HTTP_200_OK)
+        return Response(status=status.HTTP_400_BAD_REQUEST, data={"error": "Division is required."})
+
 
 class CandidateURLBulkCreateView(generics.ListCreateAPIView):
     queryset = CandidateURL.objects.all()
@@ -386,6 +397,21 @@ def create(self, request, *args, **kwargs):
                 return Response(status=status.HTTP_204_NO_CONTENT)
 
 
+class DivisionPatternViewSet(CollectionFilterMixin, viewsets.ModelViewSet):
+    queryset = DivisionPattern.objects.all()
+    serializer_class = DivisionPatternSerializer
+
+    def get_queryset(self):
+        return super().get_queryset().order_by("match_pattern")
+
+    def create(self, request, *args, **kwargs):
+        division = request.POST.get("division")
+        if division:
+            return super().create(request, *args, **kwargs)
+        else:
+            return Response(status=status.HTTP_400_BAD_REQUEST, data={"error": "Division is required."})
+
+
 class CollectionViewSet(viewsets.ModelViewSet):
     queryset = Collection.objects.all()
     serializer_class = CollectionSerializer

sde_indexing_helper/static/css/candidate_url_list.css

@@ -334,7 +334,7 @@ div.dt-buttons .btn.processing:after {
     -webkit-animation: dtb-spinner 1500ms infinite linear;
 }
 
-.document_type_dropdown, .dropdown-toggle {
+.document_type_dropdown, .division_dropdown, .dropdown-toggle {
     width: 100%;
     display: flex;
     justify-content: center;