fix postgres metadata col naming and channel init

Author: lalepee

cosmotech/coal/postgresql/runner.py

@@ -51,7 +51,7 @@ def send_runner_metadata_to_postgresql(
                 CREATE TABLE IF NOT EXISTS {schema_table}  (
                   id varchar(32) PRIMARY KEY,
                   name varchar(256),
-                  last_run_id varchar(32),
+                  last_csm_run_id varchar(32),
                   run_template_id varchar(32)
                 );
             """
@@ -60,11 +60,11 @@ def send_runner_metadata_to_postgresql(
             conn.commit()
             LOGGER.info(T("coal.services.postgresql.metadata"))
             sql_upsert = f"""
-                INSERT INTO {schema_table} (id, name, last_run_id, run_template_id)
+                INSERT INTO {schema_table} (id, name, last_csm_run_id, run_template_id)
                   VALUES(%s, %s, %s, %s)
                   ON CONFLICT (id)
                   DO
-                    UPDATE SET name = EXCLUDED.name, last_run_id = EXCLUDED.last_run_id;
+                    UPDATE SET name = EXCLUDED.name, last_csm_run_id = EXCLUDED.last_csm_run_id;
             """
             curs.execute(
                 sql_upsert,
@@ -108,7 +108,7 @@ def remove_runner_metadata_from_postgresql(
             schema_table = f"{_psql.db_schema}.{_psql.table_prefix}RunnerMetadata"
             sql_delete_from_metatable = f"""
                 DELETE FROM {schema_table}
-                WHERE last_run_id={runner.get("lastRunId")};
+                WHERE last_csm_run_id={runner.get("lastRunId")};
             """
             curs.execute(sql_delete_from_metatable)
             conn.commit()

cosmotech/coal/postgresql/store.py

@@ -86,7 +86,6 @@ def dump_store_to_postgresql_from_conf(
         fk_id: foreign key id to add to all table on all rows
     """
     _psql = PostgresUtils(configuration)
-    print(_psql.send_pyarrow_table_to_postgresql)
     _s = Store(store_location=store_folder)
 
     tables = list(_s.list_tables())
@@ -103,9 +102,9 @@ def dump_store_to_postgresql_from_conf(
             if fk_id:
                 _s.execute_query(
                     f"""
-                    ALTER TABLE {_psql.table_prefix}{table_name}
-                    ADD run_id TEXT NOT NULL
-                    DEFAULT ({fk_id})
+                    ALTER TABLE {table_name}
+                    ADD csm_run_id TEXT NOT NULL
+                    DEFAULT ('{fk_id})
                     """
                 )
             data = _s.get_table(table_name)
@@ -120,7 +119,7 @@ def dump_store_to_postgresql_from_conf(
             )
             if fk_id and _psql.is_metadata_exists():
                 metadata_table = f"{_psql.table_prefix}RunnerMetadata"
-                _psql.add_fk_constraint(table_name, "run_id", metadata_table, "last_run_id")
+                _psql.add_fk_constraint(table_name, "csm_run_id", metadata_table, "last_csm_run_id")
 
             total_rows += rows
             _up_time = perf_counter()

cosmotech/coal/store/output/aws_channel.py

@@ -8,7 +8,7 @@
 from cosmotech.coal.aws import S3
 from cosmotech.coal.store.output.channel_interface import ChannelInterface
 from cosmotech.coal.store.store import Store
-from cosmotech.coal.utils.configuration import Configuration
+from cosmotech.coal.utils.configuration import Configuration, Dotdict
 from cosmotech.coal.utils.logger import LOGGER
 
 
@@ -21,7 +21,7 @@ class AwsChannel(ChannelInterface):
     }
     requirement_string = required_keys
 
-    def __init__(self, dct: dict = None):
+    def __init__(self, dct: Dotdict = None):
         self.configuration = Configuration(dct)
         self._s3 = S3(self.configuration)
 

cosmotech/coal/store/output/az_storage_channel.py

@@ -2,7 +2,7 @@
 
 from cosmotech.coal.azure.blob import dump_store_to_azure
 from cosmotech.coal.store.output.channel_interface import ChannelInterface
-from cosmotech.coal.utils.configuration import Configuration
+from cosmotech.coal.utils.configuration import Configuration, Dotdict
 
 
 class AzureStorageChannel(ChannelInterface):
@@ -22,7 +22,7 @@ class AzureStorageChannel(ChannelInterface):
     }
     requirement_string = required_keys
 
-    def __init__(self, dct: dict = None):
+    def __init__(self, dct: Dotdict = None):
         self.configuration = Configuration(dct)
 
     def send(self, tables_filter: Optional[list[str]] = None) -> bool:

cosmotech/coal/store/output/postgres_channel.py

@@ -6,7 +6,7 @@
 )
 from cosmotech.coal.postgresql.store import dump_store_to_postgresql_from_conf
 from cosmotech.coal.store.output.channel_interface import ChannelInterface
-from cosmotech.coal.utils.configuration import Configuration
+from cosmotech.coal.utils.configuration import Configuration, Dotdict
 
 
 class PostgresChannel(ChannelInterface):
@@ -23,7 +23,7 @@ class PostgresChannel(ChannelInterface):
     }
     requirement_string = required_keys
 
-    def __init__(self, dct: dict = None):
+    def __init__(self, dct: Dotdict = None):
         self.configuration = Configuration(dct)
 
     def send(self, tables_filter: Optional[list[str]] = None) -> bool: