NRL-1346 Improved error handling

Author: jackleary

terraform/account-wide-infrastructure/modules/glue/src/pipeline.py

@@ -118,6 +118,10 @@ def load(self, data):
             if name == "s2c":
                 name = "ssp"
             try:
+                if dataframe.rdd.isEmpty():
+                    self.logger.info(f"{name} dataframe has no rows. Skipping.")
+                    continue
+
                 self.logger.info(
                     f"Attempting to load dataframe {name} into {self.target_path}{name}"
                 )