Added hydroserverpy ETL task, ETL tests, and ETL migration

Author: Ken Lippold

api/urls.py

@@ -49,22 +49,10 @@
 api.add_router("processing-levels", processing_level_router)
 api.add_router("result-qualifiers", result_qualifier_router)
 
-etl_api = NinjaAPI(
-    title="HydroServer ETL API",
-    version=__version__,
-    urls_namespace="etl",
-    docs_decorator=ensure_csrf_cookie,
-    renderer=ORJSONRenderer(),
-    throttle=[
-        AnonRateThrottle("20/s"),
-        AuthRateThrottle("20/s"),
-    ],
-)
-
-etl_api.add_router("jobs", job_router)
-etl_api.add_router("tasks", task_router)
-etl_api.add_router("tasks", task_run_router)
-etl_api.add_router("orchestration-systems", orchestration_system_router)
+api.add_router("etl-jobs", job_router)
+api.add_router("etl-tasks", task_router)
+api.add_router("etl-tasks", task_run_router)
+api.add_router("etl-orchestration-systems", orchestration_system_router)
 
 st_api_1_1 = SensorThingsAPI(
     title="HydroServer SensorThings API",
@@ -81,6 +69,5 @@
 
 urlpatterns = [
     path("data/", api.urls),
-    path("etl/", etl_api.urls),
     path("sensorthings/v1.1/", st_api_1_1.urls),
 ]

etl/management/__init__.py

@@ -0,0 +1,21 @@
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+
+
+class Command(BaseCommand):
+    help = "Loads test data from fixtures"
+
+    def handle(self, *args, **kwargs):
+        fixtures = [
+            "tests/fixtures/test_etl_jobs.yaml",
+            "tests/fixtures/test_etl_orchestration_systems.yaml",
+            "tests/fixtures/test_etl_tasks.yaml",
+        ]
+
+        for fixture in fixtures:
+            self.stdout.write(self.style.NOTICE(f"Loading fixture: {fixture}"))
+            try:
+                call_command("loaddata", fixture)
+                self.stdout.write(self.style.SUCCESS(f"Successfully loaded {fixture}"))
+            except Exception as e:
+                self.stderr.write(self.style.ERROR(f"Failed to load {fixture}: {e}"))