Merge pull request #929 from NASA-IMPACT/928-update-cosmos-to-handle-dev-and-test-ej-data

add DestinationServer Choices to EnvironmentalJusticeRow

Author: code-geek

README.md

@@ -78,7 +78,26 @@ To load collections:
 $ docker-compose -f local.yml run --rm django python manage.py loaddata sde_collections/fixtures/collections.json
 ```
 
-### Loading the Database from a Backup
+### Manually Creating and Loading a ContentTypeless Backup
+Navigate to the server running prod, then to the project folder. Run the following command to create a backup:
+
+```bash
+docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20240812.json
+```
+This will have saved the backup in a folder outside of the docker container. Now you can copy it to your local machine. 
+
+```bash
+mv ~/prod_backup-20240812.json <project_path>/prod_backup-20240812.json
+scp sde:/home/ec2-user/sde_indexing_helper/backups/prod_backup-20240812.json prod_backup-20240812.json
+```
+
+Finally, load the backup into your local database:
+
+```bash
+docker-compose -f local.yml run --rm django python manage.py loaddata prod_backup-20240812.json
+```
+
+### Loading the Database from an Arbitrary Backup
 
 1. Build the project and run the necessary containers (as documented above).
 2. Clear out content types using the Django shell:

config/settings/base.py

@@ -74,10 +74,21 @@
     "crispy_bootstrap5",
     "crispy_forms",
     "django_celery_beat",
+    "django_filters",
     "rest_framework_datatables",
     "rest_framework",
 ]
 
+LOCAL_APPS = [
+    "environmental_justice",
+    "feedback",
+    "sde_collections",
+    "sde_indexing_helper.users",
+]
+
+# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
+INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
+
 CORS_ALLOWED_ORIGINS = [
     "http://localhost:3000",
     "http://sde-lrm.nasa-impact.net",
@@ -91,15 +102,6 @@
     "http://localhost:4200",
 ]
 
-LOCAL_APPS = [
-    "environmental_justice",
-    "sde_indexing_helper.users",
-    "sde_collections",
-    "feedback",
-]
-# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
-INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
-
 # MIGRATIONS
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#migration-modules

environmental_justice/migrations/0005_environmentaljusticerow_destination_server.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+def set_default_destination_server(apps, schema_editor):
+    EnvironmentalJusticeRow = apps.get_model("environmental_justice", "EnvironmentalJusticeRow")
+    EnvironmentalJusticeRow.objects.filter(destination_server="").update(destination_server="prod")
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("environmental_justice", "0004_alter_environmentaljusticerow_data_visualization_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="environmentaljusticerow",
+            name="destination_server",
+            field=models.CharField(
+                blank=True,
+                choices=[("dev", "Development"), ("test", "Testing"), ("prod", "Production")],
+                default="",
+                max_length=10,
+                verbose_name="Destination Server",
+            ),
+        ),
+        migrations.RunPython(set_default_destination_server),
+    ]

environmental_justice/models.py

@@ -6,11 +6,18 @@ class EnvironmentalJusticeRow(models.Model):
     Environmental Justice data from the spreadsheet
     """
 
+    class DestinationServerChoices(models.TextChoices):
+        DEV = "dev", "Development"
+        TEST = "test", "Testing"
+        PROD = "prod", "Production"
+
+    destination_server = models.CharField(
+        "Destination Server", max_length=10, choices=DestinationServerChoices.choices, default="", blank=True
+    )
+
     dataset = models.CharField("Dataset", blank=True, default="")
     description = models.CharField("Description", blank=True, default="")
-    description_simplified = models.CharField(
-        "Description Simplified", blank=True, default=""
-    )
+    description_simplified = models.CharField("Description Simplified", blank=True, default="")
     indicators = models.CharField("Indicators", blank=True, default="")
     intended_use = models.CharField("Intended Use", blank=True, default="")
     latency = models.CharField("Latency", blank=True, default="")
@@ -21,15 +28,11 @@ class EnvironmentalJusticeRow(models.Model):
 
     # fields that needs cleaning
     format = models.CharField("Format", blank=True, default="")
-    geographic_coverage = models.CharField(
-        "Geographic Coverage", blank=True, default=""
-    )
+    geographic_coverage = models.CharField("Geographic Coverage", blank=True, default="")
     data_visualization = models.CharField("Data Visualization", blank=True, default="")
     spatial_resolution = models.CharField("Spatial Resolution", blank=True, default="")
     temporal_extent = models.CharField("Temporal Extent", blank=True, default="")
-    temporal_resolution = models.CharField(
-        "Temporal Resolution", blank=True, default=""
-    )
+    temporal_resolution = models.CharField("Temporal Resolution", blank=True, default="")
 
     sde_link = models.CharField("SDE Link", default="", blank=True)
 

environmental_justice/views.py

@@ -1,4 +1,5 @@
 from rest_framework import viewsets
+from django_filters.rest_framework import DjangoFilterBackend
 
 from .models import EnvironmentalJusticeRow
 from .serializers import EnvironmentalJusticeRowSerializer
@@ -11,6 +12,15 @@ class EnvironmentalJusticeRowViewSet(viewsets.ModelViewSet):
 
     queryset = EnvironmentalJusticeRow.objects.all()
     serializer_class = EnvironmentalJusticeRowSerializer
-    http_method_names = [
-        "get",
-    ]
+    http_method_names = ["get"]
+    filter_backends = [DjangoFilterBackend]
+    filterset_fields = ["destination_server"]
+
+    def get_queryset(self):
+        """
+        if no destination_server is provided, default to PROD
+        """
+        queryset = super().get_queryset()
+        if not self.request.query_params.get("destination_server"):
+            queryset = queryset.filter(destination_server=EnvironmentalJusticeRow.DestinationServerChoices.PROD)
+        return queryset

requirements/base.txt

@@ -21,11 +21,12 @@ crispy-bootstrap5==2023.10  # https://github.com/django-crispy-forms/crispy-boot
 # project-specific
 beautifulsoup4==4.12.3
 boto3==1.34.31
+django-cors-headers==4.3.1
+django-filter==24.3
 djangorestframework-datatables==0.7.0
 djangorestframework==3.14.0
+lxml==4.9.2
 PyGithub==2.2.0
 tqdm==4.66.3
-xmltodict==0.13.0
-django-cors-headers==4.3.1
 unidecode==1.3.8
-lxml==4.9.2
+xmltodict==0.13.0