lint

Author: pooleycodes

request-processor/src/application/core/pipeline.py

@@ -7,8 +7,6 @@
 
 from digital_land.pipeline import Pipeline, Lookups
 from digital_land.commands import get_resource_unidentified_lookups
-from application.core.utils import append_endpoint, append_source
-from datetime import datetime
 from pathlib import Path
 
 logger = get_logger(__name__)
@@ -42,7 +40,7 @@ def fetch_response_data(
     try:
         for file_name in files_in_resource:
             file_path = os.path.join(input_path, file_name)
-            # retrieve unnassigned entities and assign
+            # retrieve unnassigned entities and assign, TODO: Is this necessary here?
             assign_entries(
                 resource_path=file_path,
                 dataset=dataset,
@@ -197,7 +195,7 @@ def fetch_add_data_response(
     output_path,
     specification_dir,
     cache_dir,
-    url
+    url,
 ):
     try:
         specification = Specification(specification_dir)
@@ -262,7 +260,9 @@ def fetch_add_data_response(
                     new_entities.extend(new_lookups)
 
                     # Reload pipeline to pick up newly saved lookups
-                    pipeline = Pipeline(pipeline_dir, dataset, specification=specification)
+                    pipeline = Pipeline(
+                        pipeline_dir, dataset, specification=specification
+                    )
 
                     # Now re-run transform to check and return issue log
                     issues_log = pipeline.transform(
@@ -401,4 +401,4 @@ def _map_transformed_entities(transformed_csv_path, pipeline_dir):  # noqa: C901
                 }
             )
 
-    return mapped_entities
+    return mapped_entities

request-processor/src/application/core/workflow.py

@@ -453,10 +453,14 @@ def add_data_workflow(
 
         # Loads csvs for Pipeline and Config
         if not fetch_add_data_pipeline_csvs(collection, pipeline_dir):
-            response_data['message'] = f"Unable to find lookups for collection '{collection}', dataset '{dataset}'"
+            response_data[
+                "message"
+            ] = f"Unable to find lookups for collection '{collection}', dataset '{dataset}'"
             return response_data
         if not fetch_add_data_collection_csvs(collection, collection_dir):
-            response_data['message'] = f"Unable to find lookups for collection '{collection}', dataset '{dataset}'"
+            response_data[
+                "message"
+            ] = f"Unable to find lookups for collection '{collection}', dataset '{dataset}'"
             return response_data
 
         # All processes arount transforming the data and generating pipeline summary
@@ -468,7 +472,7 @@ def add_data_workflow(
             output_path=output_path,
             specification_dir=directories.SPECIFICATION_DIR,
             cache_dir=directories.CACHE_DIR,
-            url=url
+            url=url,
         )
 
         # Create endpoint and source summaries in workflow
@@ -498,8 +502,10 @@ def add_data_workflow(
         logger.info(f"add data response is for id {request_id} : {response_data}")
 
     except Exception as e:
-        logger.warning(f"An error occurred in add_data_workflow: {e} for request id {request_id}")
-        response_data['message'] = f"An error occurred in add_data_workflow: {e}"
+        logger.warning(
+            f"An error occurred in add_data_workflow: {e} for request id {request_id}"
+        )
+        response_data["message"] = f"An error occurred in add_data_workflow: {e}"
 
     finally:
         clean_up(