Merge branch 'dev' of https://github.com/NASA-IMPACT/COSMOS into 1052-update-cosmos-to-create-jobs-for-scrapers-and-indexers

Author: dhanur-sharma

.envs/.local/.django

@@ -33,9 +33,17 @@ SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH='dummy_branch'
 # Slack Webhook
 # ------------------------------------------------------------------------------
 SLACK_WEBHOOK_URL=''
-LRM_USER=''
-LRM_PASSWORD=''
+
+#Server Credentials
+#--------------------------------------------------------------------------------
+LRM_DEV_USER=''
+LRM_DEV_PASSWORD=''
 XLI_USER=''
 XLI_PASSWORD=''
 LRM_QA_USER=''
 LRM_QA_PASSWORD=''
+
+#Server Tokens
+#--------------------------------------------------------------------------------
+LRM_DEV_TOKEN=''
+XLI_TOKEN=''

README.md

@@ -82,7 +82,7 @@ $ docker-compose -f local.yml run --rm django python manage.py loaddata sde_coll
 Navigate to the server running prod, then to the project folder. Run the following command to create a backup:
 
 ```bash
-docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20240812.json
+docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20241114.json
 ```
 This will have saved the backup in a folder outside of the docker container. Now you can copy it to your local machine.
 
@@ -208,3 +208,24 @@ Eventually, job creation will be done seamlessly by the webapp. Until then, edit
   - JavaScript: `/sde_indexing_helper/static/js`
   - CSS: `/sde_indexing_helper/static/css`
   - Images: `/sde_indexing_helper/static/images`
+
+
+## Running Long Scripts on the Server
+```shell
+tmux new -s docker_django
+```
+Once you are inside, you can run dmshell or for example a managment command:
+
+```shell
+docker-compose -f production.yml run --rm django python manage.py deduplicate_urls
+```
+
+Later, you can do this to get back in.
+```shell
+tmux attach -t docker_django
+```
+
+To delete the session:
+```shell
+tmux kill-session -t docker_django
+```

compose/production/traefik/traefik.yml

@@ -31,7 +31,7 @@ certificatesResolvers:
 http:
   routers:
     web-secure-router:
-      rule: "Host(`sde-indexing-helper.nasa-impact.net`)"
+      rule: 'Host(`{{ env "TRAEFIK_DOMAIN" }}`)'
       entryPoints:
         - web-secure
       middlewares:
@@ -42,7 +42,7 @@ http:
         certResolver: letsencrypt
 
     flower-secure-router:
-      rule: "Host(`sde-indexing-helper.nasa-impact.net`)"
+      rule: 'Host(`{{ env "TRAEFIK_DOMAIN" }}`)'
       entryPoints:
         - flower
       service: flower

config/settings/base.py

@@ -343,7 +343,9 @@
 SLACK_WEBHOOK_URL = env("SLACK_WEBHOOK_URL")
 XLI_USER = env("XLI_USER")
 XLI_PASSWORD = env("XLI_PASSWORD")
-LRM_USER = env("LRM_USER")
-LRM_PASSWORD = env("LRM_PASSWORD")
+LRM_DEV_USER = env("LRM_DEV_USER")
+LRM_DEV_PASSWORD = env("LRM_DEV_PASSWORD")
 LRM_QA_USER = env("LRM_QA_USER")
 LRM_QA_PASSWORD = env("LRM_QA_PASSWORD")
+LRM_DEV_TOKEN = env("LRM_DEV_TOKEN")
+XLI_TOKEN = env("XLI_TOKEN")

environmental_justice/README.md

@@ -0,0 +1,86 @@
+# Environmental Justice API
+
+## Overview
+This API provides access to Environmental Justice data from multiple sources. It supports retrieving data from individual sources or as a combined dataset with defined precedence rules.
+
+## Endpoints
+
+### GET /api/environmental-justice/
+
+Retrieves environmental justice data based on specified data source.
+
+#### Query Parameters
+
+| Parameter    | Description | Default    | Options                                      |
+|-------------|-------------|------------|----------------------------------------------|
+| data_source | Data source filter | "combined" | "spreadsheet", "ml_production", "ml_testing", "combined" |
+
+#### Data Source Behavior
+
+1. **Single Source**
+   - `?data_source=spreadsheet`: Returns only spreadsheet data
+   - `?data_source=ml_production`: Returns only ML production data
+   - `?data_source=ml_testing`: Returns only ML testing data
+
+2. **Combined Data** (Default)
+   - Access via `?data_source=combined` or no parameter
+   - Merges data from 'spreadsheet' and 'ml_production' sources
+   - Precedence rules:
+     - If the same dataset exists in both sources, the spreadsheet version is used
+     - Unique datasets from ml_production are included
+     - ML testing data is not included in combined view
+
+#### Example Requests
+
+```bash
+# Get combined data (default)
+GET /api/environmental-justice/
+
+# Get combined data (explicit)
+GET /api/environmental-justice/?data_source=combined
+
+# Get only spreadsheet data
+GET /api/environmental-justice/?data_source=spreadsheet
+
+# Get only ML production data
+GET /api/environmental-justice/?data_source=ml_production
+
+# Get only ML testing data
+GET /api/environmental-justice/?data_source=ml_testing
+```
+
+#### Response Fields
+
+Each record includes the following fields:
+- dataset
+- description
+- description_simplified
+- indicators
+- intended_use
+- latency
+- limitations
+- project
+- source_link
+- strengths
+- format
+- geographic_coverage
+- data_visualization
+- spatial_resolution
+- temporal_extent
+- temporal_resolution
+- sde_link
+- data_source
+
+## Data Source Definitions
+
+- **spreadsheet**: Primary source data from environmental justice spreadsheets
+- **ml_production**: Production machine learning processed data
+- **ml_testing**: Testing/staging machine learning processed data
+
+## Precedence Rules
+When retrieving combined data:
+1. If a dataset exists in both spreadsheet and ml_production:
+   - The spreadsheet version takes precedence
+   - The ml_production version is excluded
+2. Datasets unique to ml_production are included in the response
+3. ML testing data is never included in combined results

environmental_justice/migrations/0006_remove_environmentaljusticerow_destination_server_and_more.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.2.9 on 2024-11-23 03:18
+
+from django.db import migrations, models
+
+
+def migrate_destination_server_to_data_source(apps, schema_editor):
+    EnvironmentalJusticeRow = apps.get_model("environmental_justice", "EnvironmentalJusticeRow")
+
+    # Migrate prod to spreadsheet
+    EnvironmentalJusticeRow.objects.filter(destination_server="prod").update(
+        data_source="spreadsheet", destination_server=""
+    )
+
+    # Migrate dev to ml_production
+    EnvironmentalJusticeRow.objects.filter(destination_server="dev").update(
+        data_source="ml_production", destination_server=""
+    )
+
+    # Migrate test to ml_testing
+    EnvironmentalJusticeRow.objects.filter(destination_server="test").update(
+        data_source="ml_testing", destination_server=""
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("environmental_justice", "0005_environmentaljusticerow_destination_server"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="environmentaljusticerow",
+            name="data_source",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("spreadsheet", "Spreadsheet"),
+                    ("ml_production", "ML Production"),
+                    ("ml_testing", "ML Testing"),
+                ],
+                default="",
+                max_length=20,
+                verbose_name="Data Source",
+            ),
+        ),
+        migrations.RunPython(migrate_destination_server_to_data_source, reverse_code=migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="environmentaljusticerow",
+            name="destination_server",
+        ),
+    ]

environmental_justice/models.py

@@ -6,13 +6,13 @@ class EnvironmentalJusticeRow(models.Model):
     Environmental Justice data from the spreadsheet
     """
 
-    class DestinationServerChoices(models.TextChoices):
-        DEV = "dev", "Development"
-        TEST = "test", "Testing"
-        PROD = "prod", "Production"
+    class DataSourceChoices(models.TextChoices):
+        SPREADSHEET = "spreadsheet", "Spreadsheet"
+        ML_PRODUCTION = "ml_production", "ML Production"
+        ML_TESTING = "ml_testing", "ML Testing"
 
-    destination_server = models.CharField(
-        "Destination Server", max_length=10, choices=DestinationServerChoices.choices, default="", blank=True
+    data_source = models.CharField(
+        "Data Source", max_length=20, choices=DataSourceChoices.choices, default="", blank=True
     )
 
     dataset = models.CharField("Dataset", blank=True, default="")

environmental_justice/tests.py

@@ -0,0 +1,30 @@
+import pytest
+from django.urls import include, path
+from rest_framework.routers import DefaultRouter
+from rest_framework.test import APIClient
+
+from environmental_justice.views import EnvironmentalJusticeRowViewSet
+
+# Create router and register our viewset
+router = DefaultRouter()
+router.register(r"environmental-justice", EnvironmentalJusticeRowViewSet)
+
+# Create temporary urlpatterns for testing
+urlpatterns = [
+    path("api/", include(router.urls)),
+]
+
+
+# Override default URL conf for testing
+@pytest.fixture
+def client():
+    """Return a Django REST framework API client"""
+    return APIClient()
+
+
+@pytest.fixture(autouse=True)
+def setup_urls():
+    """Setup URLs for testing"""
+    from django.conf import settings
+
+    settings.ROOT_URLCONF = __name__

environmental_justice/tests/conftest.py

@@ -0,0 +1,28 @@
+import factory
+from factory.django import DjangoModelFactory
+
+from environmental_justice.models import EnvironmentalJusticeRow
+
+
+class EnvironmentalJusticeRowFactory(DjangoModelFactory):
+    class Meta:
+        model = EnvironmentalJusticeRow
+
+    dataset = factory.Sequence(lambda n: f"dataset_{n}")
+    description = factory.Faker("sentence")
+    description_simplified = factory.Faker("sentence")
+    indicators = factory.Faker("sentence")
+    intended_use = factory.Faker("sentence")
+    latency = factory.Faker("word")
+    limitations = factory.Faker("sentence")
+    project = factory.Faker("word")
+    source_link = factory.Faker("url")
+    strengths = factory.Faker("sentence")
+    format = factory.Faker("file_extension")
+    geographic_coverage = factory.Faker("country")
+    data_visualization = factory.Faker("sentence")
+    spatial_resolution = factory.Faker("word")
+    temporal_extent = factory.Faker("date")
+    temporal_resolution = factory.Faker("word")
+    sde_link = factory.Faker("url")
+    data_source = EnvironmentalJusticeRow.DataSourceChoices.SPREADSHEET