NRL-1320 Exit if df is null

Author: jackleary

terraform/account-wide-infrastructure/modules/glue/src/pipeline.py

@@ -90,12 +90,11 @@ def load(self, data):
         """Load transformed data into Parquet format"""
         self.logger.info(f"Loading data into {self.target_path} as Parquet")
         for name, dataframe in data.items():
-            if not dataframe:
-                continue
-            name = name.replace("--", "_")
-            dataframe.write.mode("append").partitionBy(*self.partition_cols).parquet(
-                f"{self.target_path}{name}"
-            )
+            if dataframe.na.drop().count() > 0:
+                name = name.replace("--", "_")
+                dataframe.write.mode("append").partitionBy(
+                    *self.partition_cols
+                ).parquet(f"{self.target_path}{name}")
 
     def trigger_crawler(self):
         self.glue.start_crawler(Name=f"{self.name_prefix}-log-crawler")