NRL-1411 attempt to improve effeciency

Author: jackleary

terraform/account-wide-infrastructure/modules/glue/src/pipeline.py

@@ -104,9 +104,9 @@ def load(self, data):
         self.logger.info(f"Loading data into {self.target_path} as Parquet")
         for name, dataframe in data.items():
             name = name.replace("--", "_")
-            dataframe.write.mode("append").partitionBy(*self.partition_cols).parquet(
-                f"{self.target_path}{name}"
-            )
+            dataframe.coalesce(1).write.mode("append").partitionBy(
+                *self.partition_cols
+            ).parquet(f"{self.target_path}{name}")
 
     def trigger_crawler(self):
         self.glue.start_crawler(Name=f"{self.name_prefix}-log-crawler")