Enabled HydroShare archival

Author: Ken Lippold

etl/fixtures/hydroshare_archival.yaml

@@ -13,6 +13,7 @@
   pk: 529b62fc-84bf-4173-8add-c9fc9126d3db
   fields:
     name: HydroShare Archive Loader Settings
+    etl_system_platform: eb1a26e0-75c9-415d-9895-8b0be8466139
     etl_configuration_model: datasource
     etl_configuration_type: loader
     etl_configuration_schema:

etl/models/etl_configuration.py

@@ -53,17 +53,17 @@ class EtlConfiguration(models.Model, PermissionChecker):
     @property
     def data_sources(self):
         return DataSource.objects.filter(
-            models.Q(extractor_configuration=self) |
-            models.Q(transformer_configuration=self) |
-            models.Q(loader_configuration=self)
+            models.Q(extractor_configuration=self)
+            | models.Q(transformer_configuration=self)
+            | models.Q(loader_configuration=self)
         ).distinct()
 
     @property
     def linked_datastreams(self):
         return LinkedDatastream.objects.filter(
-            models.Q(extractor_configuration=self) |
-            models.Q(transformer_configuration=self) |
-            models.Q(loader_configuration=self)
+            models.Q(extractor_configuration=self)
+            | models.Q(transformer_configuration=self)
+            | models.Q(loader_configuration=self)
         ).distinct()
 
     objects = EtlConfigurationQuerySet.as_manager()
@@ -99,9 +99,14 @@ def delete_contents(filter_arg: models.Model, filter_suffix: Optional[str]):
 
         if filter_suffix:
             etl_configuration_relation_filters = [
-                f"{relation}__{filter_suffix}" for relation in etl_configuration_relation_filters
+                f"{relation}__{filter_suffix}"
+                for relation in etl_configuration_relation_filters
             ]
 
         for relation in etl_configuration_relation_filters:
-            DataSource.objects.filter(**{relation: filter_arg}).update(**{relation: None})
-            LinkedDatastream.objects.filter(**{relation: filter_arg}).update(**{relation: None})
+            DataSource.objects.filter(**{relation: filter_arg}).update(
+                **{relation: None}
+            )
+            LinkedDatastream.objects.filter(**{relation: filter_arg}).update(
+                **{relation: None}
+            )

etl/schemas/__init__.py

@@ -19,7 +19,8 @@
     EtlConfigurationPatchBody,
 )
 from .hydroshare_archival import (
+    HydroShareArchivalFields,
     HydroShareArchivalGetResponse,
     HydroShareArchivalPostBody,
-    HydroShareArchivalPatchBody
+    HydroShareArchivalPatchBody,
 )

etl/schemas/hydroshare_archival.py

@@ -7,7 +7,7 @@
 
 class HydroShareArchivalFields(Schema):
     link: Optional[AnyHttpUrl] = None
-    frequency: Optional[Literal["daily", "weekly", "monthly"]]
+    frequency: Optional[Literal["daily", "weekly", "monthly"]] = None
     path: str = Field(..., max_length=255)
     datastream_ids: list[UUID]
 

etl/services/etl_configuration.py

@@ -159,16 +159,16 @@ def delete(self, user: User, etl_system_platform_id: uuid.UUID, uid: uuid.UUID):
         )
 
         if DataSource.objects.filter(
-            Q(extractor_configuration=etl_configuration) |
-            Q(transformer_configuration=etl_configuration) |
-            Q(loader_configuration=etl_configuration)
+            Q(extractor_configuration=etl_configuration)
+            | Q(transformer_configuration=etl_configuration)
+            | Q(loader_configuration=etl_configuration)
         ).exists():
             raise HttpError(409, "ETL configuration in use by one or more data sources")
 
         if LinkedDatastream.objects.filter(
-            Q(extractor_configuration=etl_configuration) |
-            Q(transformer_configuration=etl_configuration) |
-            Q(loader_configuration=etl_configuration)
+            Q(extractor_configuration=etl_configuration)
+            | Q(transformer_configuration=etl_configuration)
+            | Q(loader_configuration=etl_configuration)
         ).exists():
             raise HttpError(409, "ETL configuration in use by one or more datastreams")