Merge pull request #1125 from NASA-IMPACT/1055-add-additional-workflow-statuses-on-cosmos-to-help-the-re-indexation-process

add initial reindexing statuses

Author: CarsonDavis

sde_collections/admin.py

@@ -11,7 +11,7 @@
 )
 
 from .models.candidate_url import CandidateURL, ResolvedTitle
-from .models.collection import Collection, WorkflowHistory
+from .models.collection import Collection, ReindexingHistory, WorkflowHistory
 from .models.collection_choice_fields import TDAMMTags
 from .models.delta_url import CuratedUrl, DeltaUrl, DumpUrl
 from .models.pattern import DivisionPattern, IncludePattern, TitlePattern
@@ -217,6 +217,7 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
                     "source",
                     "turned_on",
                     "is_multi_division",
+                    "reindexing_status",
                 ),
             },
         ),
@@ -250,10 +251,18 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
         "division",
         "new_collection",
         "is_multi_division",
+        "reindexing_status",
     )
 
     readonly_fields = ("config_folder",)
-    list_filter = ("division", "curation_status", "workflow_status", "turned_on", "is_multi_division")
+    list_filter = (
+        "division",
+        "curation_status",
+        "workflow_status",
+        "turned_on",
+        "is_multi_division",
+        "reindexing_status",
+    )
     search_fields = ("name", "url", "config_folder")
     actions = [
         generate_deployment_message,
@@ -392,6 +401,12 @@ class WorkflowHistoryAdmin(admin.ModelAdmin):
     list_filter = ["workflow_status", "old_status"]
 
 
+class ReindexingHistoryAdmin(admin.ModelAdmin):
+    list_display = ("collection", "old_status", "reindexing_status", "created_at")
+    search_fields = ["collection__name"]
+    list_filter = ["reindexing_status", "old_status"]
+
+
 class ResolvedTitleAdmin(admin.ModelAdmin):
     list_display = ["title_pattern", "candidate_url", "resolved_title", "created_at"]
 
@@ -449,6 +464,7 @@ class CuratedUrlAdmin(TDAMMAdminMixin, admin.ModelAdmin):
     form = CuratedURLForm
 
 
+admin.site.register(ReindexingHistory, ReindexingHistoryAdmin)
 admin.site.register(WorkflowHistory, WorkflowHistoryAdmin)
 admin.site.register(CandidateURL, CandidateURLAdmin)
 admin.site.register(TitlePattern, TitlePatternAdmin)

sde_collections/migrations/0072_collection_reindexing_status_reindexinghistory.py

@@ -0,0 +1,167 @@
+# Generated by Django 4.2.9 on 2024-12-06 03:51
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def set_initial_reindexing_status(apps, schema_editor):
+    Collection = apps.get_model("sde_collections", "Collection")
+
+    # List of collections that have been reindexed on LRM dev
+    reindexed_collections = {
+        "astrophysics_source_code_library",
+        "astrophysics_science_division_asd_code_660",
+        "the_astrophysics_astrochemistry_lab",
+        "Space_Physics_Data_Facility",
+        "ppi_node",
+        "sun_climate_powered_by_solar_irradiance",
+        "magnetospheric_multiscale_satellites",
+        "mdscc_deep_space_network",
+        "voyager",
+        "f_prime",
+        "interactive_multiinstrument_database_of_solar_flares",
+        "cii_hosted_payload_opportunity_online_database",
+        "national_space_weather_program",
+        "starchild_a_learning_center_for_young_astronomers",
+        "nexsci",
+        "explorer_1",
+        "the_new_great_observatories",
+        "nasa_ames_intelligent_systems_division_data",
+        "tropical_cyclone_information_system_data_repository",
+        "explorers_and_heliophysics_projects_division",
+    }
+
+    # Define the workflow status values
+    RESEARCH_IN_PROGRESS = 1
+    READY_FOR_ENGINEERING = 2
+    ENGINEERING_IN_PROGRESS = 3
+    READY_FOR_CURATION = 4
+    CURATION_IN_PROGRESS = 5
+    CURATED = 6
+    QUALITY_FIXED = 7
+    SECRET_DEPLOYMENT_STARTED = 8
+    SECRET_DEPLOYMENT_FAILED = 9
+    READY_FOR_LRM_QUALITY_CHECK = 10
+    READY_FOR_FINAL_QUALITY_CHECK = 11
+    QUALITY_CHECK_FAILED = 12
+    QUALITY_CHECK_PERFECT = 13
+    MERGE_PENDING = 14
+    NEEDS_DELETE = 19
+
+    # Workflow statuses that should be marked as reindexing not needed
+    reindexing_not_needed_statuses = [
+        RESEARCH_IN_PROGRESS,
+        READY_FOR_ENGINEERING,
+        ENGINEERING_IN_PROGRESS,
+        READY_FOR_CURATION,
+        CURATION_IN_PROGRESS,
+        CURATED,
+        QUALITY_FIXED,
+        SECRET_DEPLOYMENT_STARTED,
+        SECRET_DEPLOYMENT_FAILED,
+        READY_FOR_LRM_QUALITY_CHECK,
+        READY_FOR_FINAL_QUALITY_CHECK,
+        QUALITY_CHECK_FAILED,
+        QUALITY_CHECK_PERFECT,
+        MERGE_PENDING,
+        NEEDS_DELETE,
+    ]
+
+    # Set collections that have been reindexed
+    Collection.objects.filter(config_folder__in=reindexed_collections).update(reindexing_status=3)  # FINISHED
+
+    # Set collections that don't need reindexing
+    Collection.objects.filter(workflow_status__in=reindexing_not_needed_statuses).exclude(
+        config_folder__in=reindexed_collections
+    ).update(
+        reindexing_status=1
+    )  # NOT_NEEDED
+
+    # All other collections need reindexing
+    Collection.objects.exclude(config_folder__in=reindexed_collections).exclude(
+        workflow_status__in=reindexing_not_needed_statuses
+    ).update(
+        reindexing_status=2
+    )  # NEEDED
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("sde_collections", "0071_alter_candidateurl_tdamm_tag_manual_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="collection",
+            name="reindexing_status",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Reindexing Not Needed"),
+                    (2, "Reindexing Needed on LRM Dev"),
+                    (3, "Reindexing Finished on LRM Dev"),
+                    (4, "Ready for Curation"),
+                    (5, "Curated"),
+                    (6, "Indexed on Prod"),
+                ],
+                default=1,
+                verbose_name="Reindexing Status",
+            ),
+        ),
+        migrations.CreateModel(
+            name="ReindexingHistory",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                (
+                    "reindexing_status",
+                    models.IntegerField(
+                        choices=[
+                            (1, "Reindexing Not Needed"),
+                            (2, "Reindexing Needed on LRM Dev"),
+                            (3, "Reindexing Finished on LRM Dev"),
+                            (4, "Ready for Curation"),
+                            (5, "Curated"),
+                            (6, "Indexed on Prod"),
+                        ],
+                        default=1,
+                    ),
+                ),
+                (
+                    "old_status",
+                    models.IntegerField(
+                        choices=[
+                            (1, "Reindexing Not Needed"),
+                            (2, "Reindexing Needed on LRM Dev"),
+                            (3, "Reindexing Finished on LRM Dev"),
+                            (4, "Ready for Curation"),
+                            (5, "Curated"),
+                            (6, "Indexed on Prod"),
+                        ],
+                        null=True,
+                    ),
+                ),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "collection",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="reindexing_history",
+                        to="sde_collections.collection",
+                    ),
+                ),
+                (
+                    "curated_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+        ),
+        migrations.RunPython(set_initial_reindexing_status),
+    ]

sde_collections/models/README_REINDEXING_STATUSES.md

@@ -0,0 +1,55 @@
+# Reindexing Status Documentation
+
+### Reindexing Not Needed
+- Variable name: `REINDEXING_NOT_NEEDED` (1)
+- Default status for new collections
+- Applied to collections in early workflow stages (research, engineering, etc.)
+
+### Reindexing Needed on LRM Dev
+- Variable name: `REINDEXING_NEEDED_ON_DEV` (2)
+- Indicates collections that need to be reindexed on LRM Dev environment
+- For collections that have already been indexed on production
+
+### Reindexing Finished on LRM Dev
+- Variable name: `REINDEXING_FINISHED_ON_DEV` (3)
+- For collections that have completed reindexing on LRM Dev
+- Currently managed manually by LRM team via admin interface
+
+### Ready for Curation
+- Variable name: `REINDEXING_READY_FOR_CURATION` (4)
+- Automatically set when:
+  - A collection's dump URLs are migrated to delta URLs AND there are curated URLs present
+  - Triggered by Collection.migrate_dump_to_delta() method
+
+### Curated
+- Variable name: `REINDEXING_CURATED` (5)
+- Automatically set when:
+  - Delta URLs are promoted to curated URLs AND there are curated URLs present
+  - Triggered by Collection.promote_to_curated() method
+
+### Indexed on Prod
+- Variable name: `REINDEXING_INDEXED_ON_PROD` (6)
+- Currently managed manually via command line
+- Future: Will be set automatically via plugin ping
+
+### Key Code Locations for Automatic Changes
+
+1. In migrate_dump_to_delta():
+```python
+# After migrating, check if we should update reindexing status
+curated_urls_count = self.curated_urls.count()
+if curated_urls_count > 0:
+    self.reindexing_status = ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION
+    self.save()
+```
+
+2. In promote_to_curated():
+```python
+# After promoting, check if we should update reindexing status
+curated_urls_count = self.curated_urls.count()
+if curated_urls_count > 0:
+    self.reindexing_status = ReindexingStatusChoices.REINDEXING_CURATED
+    self.save()
+```
+
+Note: All status changes are logged in the ReindexingHistory model for tracking purposes.

sde_collections/models/collection.py

@@ -23,6 +23,7 @@
     CurationStatusChoices,
     Divisions,
     DocumentTypes,
+    ReindexingStatusChoices,
     SourceChoices,
     UpdateFrequencies,
     WorkflowStatusChoices,
@@ -75,6 +76,11 @@ class Collection(models.Model):
         choices=WorkflowStatusChoices.choices,
         default=WorkflowStatusChoices.RESEARCH_IN_PROGRESS,
     )
+    reindexing_status = models.IntegerField(
+        choices=ReindexingStatusChoices.choices,
+        default=ReindexingStatusChoices.REINDEXING_NOT_NEEDED,
+        verbose_name="Reindexing Status",
+    )
     tracker = FieldTracker(fields=["workflow_status"])
 
     curated_by = models.ForeignKey(User, on_delete=models.DO_NOTHING, null=True, blank=True)
@@ -155,6 +161,12 @@ def migrate_dump_to_delta(self):
         # self.refresh_url_lists_for_all_patterns() # TODO: I'm pretty confident we shouldn't be running this
         self.apply_all_patterns()
 
+        # After migrating, check if we should update reindexing status
+        curated_urls_count = self.curated_urls.count()
+        if curated_urls_count > 0:
+            self.reindexing_status = ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION
+            self.save()
+
     def create_or_update_delta_url(self, url_instance, to_delete=False):
         """
         Creates or updates a DeltaUrl entry based on the given DumpUrl or CuratedUrl object.
@@ -224,6 +236,12 @@ def promote_to_curated(self):
         # Step 4: Reapply patterns to DeltaUrls
         self.refresh_url_lists_for_all_patterns()
 
+        # After promoting, check if we should update reindexing status
+        curated_urls_count = self.curated_urls.count()
+        if curated_urls_count > 0:
+            self.reindexing_status = ReindexingStatusChoices.REINDEXING_CURATED
+            self.save()
+
     def add_to_public_query(self):
         """Add the collection to the public query."""
         if self.workflow_status not in [
@@ -349,6 +367,18 @@ def workflow_status_button_color(self) -> str:
         }
         return color_choices[self.workflow_status]
 
+    @property
+    def reindexing_status_button_color(self) -> str:
+        color_choices = {
+            1: "btn-light",  # NOT_NEEDED
+            2: "btn-warning",  # NEEDED
+            3: "btn-secondary",  # FINISHED
+            4: "btn-info",  # READY_FOR_CURATION
+            5: "btn-primary",  # CURATED
+            6: "btn-success",  # INDEXED_ON_PROD
+        }
+        return color_choices[self.reindexing_status]
+
     def _process_exclude_list(self):
         """Process the exclude list."""
         return [pattern._process_match_pattern() for pattern in self.excludepattern.all()]
@@ -654,6 +684,7 @@ def __init__(self, *args, **kwargs):
         # Create a cached version of the last workflow_status to compare against
         super().__init__(*args, **kwargs)
         self.old_workflow_status = self.workflow_status
+        self.old_reindexing_status = self.reindexing_status
 
 
 class RequiredUrls(models.Model):
@@ -734,6 +765,40 @@ def log_workflow_history(sender, instance, created, **kwargs):
             old_status=instance.old_workflow_status,
         )
 
+    if instance.reindexing_status != instance.old_reindexing_status:
+        ReindexingHistory.objects.create(
+            collection=instance,
+            reindexing_status=instance.reindexing_status,
+            curated_by=instance.curated_by,
+            old_status=instance.old_reindexing_status,
+        )
+
+
+class ReindexingHistory(models.Model):
+    collection = models.ForeignKey(Collection, on_delete=models.CASCADE, related_name="reindexing_history", null=True)
+    reindexing_status = models.IntegerField(
+        choices=ReindexingStatusChoices.choices,
+        default=ReindexingStatusChoices.REINDEXING_NOT_NEEDED,
+    )
+    old_status = models.IntegerField(choices=ReindexingStatusChoices.choices, null=True)
+    curated_by = models.ForeignKey(User, on_delete=models.DO_NOTHING, null=True, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return str(self.collection) + str(self.reindexing_status)
+
+    @property
+    def reindexing_status_button_color(self) -> str:
+        color_choices = {
+            1: "btn-light",  # REINDEXING_NOT_NEEDED
+            2: "btn-warning",  # REINDEXING_NEEDED_ON_DEV
+            3: "btn-secondary",  # REINDEXING_FINISHED_ON_DEV
+            4: "btn-info",  # REINDEXING_READY_FOR_CURATION
+            5: "btn-primary",  # REINDEXING_CURATED
+            6: "btn-success",  # REINDEXING_INDEXED_ON_PROD
+        }
+        return color_choices[self.reindexing_status]
+
 
 @receiver(post_save, sender=Collection)
 def create_configs_on_status_change(sender, instance, created, **kwargs):