Add more protection for empty tables send to external systems

Author: neomatamune

CosmoTech_Acceleration_Library/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (c) Cosmo Tech corporation.
 # Licensed under the MIT license.
 
-__version__ = '0.8.2'
+__version__ = '0.8.3'

cosmotech/coal/cli/commands/api/rds_send_store.py

@@ -80,13 +80,16 @@ def rds_send_store(
         api_run = RunApi(api_client)
         _s = Store()
         for table_name in _s.list_tables():
+            LOGGER.info(f"Sending data to table [cyan bold]CD_{table_name}[/]")
             data = convert_table_as_pylist(table_name)
+            if not len(data):
+                LOGGER.info("  - No rows : skipping")
+                continue
             fieldnames = _s.get_table_schema(table_name).names
             for row in data:
                 for field in fieldnames:
                     if row[field] is None:
                         del row[field]
-            LOGGER.info(f"Sending data to table [cyan bold]CD_{table_name}[/]")
             LOGGER.debug(f"  - Column list: {fieldnames}")
             LOGGER.info(f"  - Sending {len(data)} rows")
             api_run.send_run_data(organization_id,

cosmotech/coal/cli/commands/store/dump_to_s3.py

@@ -152,11 +152,15 @@ def data_upload(data_stream: BytesIO, file_name: str):
         for table_name in tables:
             _data_stream = BytesIO()
             _file_name = None
+            _data = _s.get_table(table_name)
+            if not len(_data):
+                LOGGER.info(f"Table {table_name} is empty (skipping)")
+                continue
             if output_type == "csv":
                 _file_name = table_name + ".csv"
-                pc.write_csv(_s.get_table(table_name), _data_stream)
+                pc.write_csv(_data, _data_stream)
             elif output_type == "parquet":
                 _file_name = table_name + ".parquet"
-                pq.write_table(_s.get_table(table_name), _data_stream)
+                pq.write_table(_data, _data_stream)
             LOGGER.info(f"Sending table {table_name} as {output_type}")
             data_upload(_data_stream, _file_name)