diff --git a/cosmotech/coal/__init__.py b/cosmotech/coal/__init__.py index 41b0e6fb..cd0520d8 100644 --- a/cosmotech/coal/__init__.py +++ b/cosmotech/coal/__init__.py @@ -5,4 +5,4 @@ # etc., to any person is prohibited unless it has been previously and # specifically authorized by written means by Cosmo Tech. -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/cosmotech/coal/postgresql/runner.py b/cosmotech/coal/postgresql/runner.py index 9ca1ad02..7ae6f3e1 100644 --- a/cosmotech/coal/postgresql/runner.py +++ b/cosmotech/coal/postgresql/runner.py @@ -32,6 +32,7 @@ def send_runner_metadata_to_postgresql( postgres_user: str, postgres_password: str, table_prefix: str = "Cosmotech_", + force_encode: bool = False, ) -> None: """ Send runner metadata to a PostgreSQL database. @@ -47,6 +48,7 @@ def send_runner_metadata_to_postgresql( postgres_user: PostgreSQL username postgres_password: PostgreSQL password table_prefix: Table prefix + force_encode: force password encoding to percent encoding """ # Get runner metadata with get_api_client()[0] as api_client: @@ -54,7 +56,7 @@ def send_runner_metadata_to_postgresql( # Generate PostgreSQL URI postgresql_full_uri = generate_postgresql_full_uri( - postgres_host, str(postgres_port), postgres_db, postgres_user, postgres_password + postgres_host, str(postgres_port), postgres_db, postgres_user, postgres_password, force_encode=force_encode ) # Connect to PostgreSQL and update runner metadata diff --git a/cosmotech/coal/postgresql/store.py b/cosmotech/coal/postgresql/store.py index 502b212f..b1e867fa 100644 --- a/cosmotech/coal/postgresql/store.py +++ b/cosmotech/coal/postgresql/store.py @@ -31,6 +31,7 @@ def dump_store_to_postgresql( postgres_password: str, table_prefix: str = "Cosmotech_", replace: bool = True, + force_encode: bool = False, ) -> None: """ Dump Store data to a PostgreSQL database. @@ -45,6 +46,7 @@ def dump_store_to_postgresql( postgres_password: PostgreSQL password table_prefix: Table prefix replace: Whether to replace existing tables + force_encode: force password encoding """ _s = Store(store_location=store_folder) @@ -72,6 +74,7 @@ def dump_store_to_postgresql( postgres_user, postgres_password, replace, + force_encode, ) total_rows += rows _up_time = perf_counter() diff --git a/cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py b/cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py index d41b87f7..cb2c4fc0 100644 --- a/cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +++ b/cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py @@ -90,6 +90,16 @@ show_envvar=True, required=True, ) +@click.option( + "--encode-password/--no-encode-password", + "force_encode", + help=T("csm_data.commands.store.postgres_send_runner_metadata.parameters.encode_password"), + envvar="CSM_PSQL_FORCE_PASSWORD_ENCODING", + show_envvar=True, + default=True, + is_flag=True, + show_default=True, +) def postgres_send_runner_metadata( organization_id, workspace_id, @@ -101,6 +111,7 @@ def postgres_send_runner_metadata( postgres_schema, postgres_user, postgres_password, + force_encode: bool, ): # Import the function at the start of the command from cosmotech.coal.postgresql import send_runner_metadata_to_postgresql @@ -116,4 +127,5 @@ def postgres_send_runner_metadata( postgres_schema=postgres_schema, postgres_user=postgres_user, postgres_password=postgres_password, + force_encode=force_encode, ) diff --git a/cosmotech/csm_data/commands/store/dump_to_postgresql.py b/cosmotech/csm_data/commands/store/dump_to_postgresql.py index 568d8e79..88ea603a 100644 --- a/cosmotech/csm_data/commands/store/dump_to_postgresql.py +++ b/cosmotech/csm_data/commands/store/dump_to_postgresql.py @@ -80,6 +80,16 @@ is_flag=True, show_default=True, ) +@click.option( + "--encode-password/--no-encode-password", + "force_encode", + help=T("csm_data.commands.store.dump_to_postgresql.parameters.encode_password"), + envvar="CSM_PSQL_FORCE_PASSWORD_ENCODING", + show_envvar=True, + default=True, + is_flag=True, + show_default=True, +) def dump_to_postgresql( store_folder, table_prefix: str, @@ -90,6 +100,7 @@ def dump_to_postgresql( postgres_user, postgres_password, replace: bool, + force_encode: bool, ): # Import the function at the start of the command from cosmotech.coal.postgresql import dump_store_to_postgresql @@ -104,4 +115,5 @@ def dump_to_postgresql( postgres_user=postgres_user, postgres_password=postgres_password, replace=replace, + force_encode=force_encode, ) diff --git a/cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml b/cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml index 21eedd77..02613b17 100644 --- a/cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml +++ b/cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml @@ -14,3 +14,4 @@ parameters: postgres_schema: PostgreSQL schema name postgres_user: PostgreSQL connection user name postgres_password: PostgreSQL connection password + encode_password: Force encoding of password to percent encoding diff --git a/cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml b/cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml index ba6a2041..de3ced66 100644 --- a/cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml +++ b/cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml @@ -17,3 +17,4 @@ parameters: postgres_user: PostgreSQL connection user name postgres_password: PostgreSQL connection password replace: Append data on existing tables + encode_password: Force encoding of password to percent encoding diff --git a/requirements.txt b/requirements.txt index e830c487..e25cac5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,10 +9,6 @@ tenacity~=8.3.0 # Keycloak connection python-keycloak~=4.7.3 -# Modelops requirements -redis==4.4.4 -redisgraph_bulk_loader==0.10.2 - # Cosmotech specific requirements cosmotech-api~=3.2 diff --git a/tests/unit/coal/test_postgresql/test_postgresql_runner.py b/tests/unit/coal/test_postgresql/test_postgresql_runner.py index c5895061..75b7c241 100644 --- a/tests/unit/coal/test_postgresql/test_postgresql_runner.py +++ b/tests/unit/coal/test_postgresql/test_postgresql_runner.py @@ -83,7 +83,7 @@ def test_send_runner_metadata_to_postgresql( # Check that PostgreSQL URI was generated correctly mock_generate_uri.assert_called_once_with( - postgres_host, str(postgres_port), postgres_db, postgres_user, postgres_password + postgres_host, str(postgres_port), postgres_db, postgres_user, postgres_password, force_encode=False ) # Check that PostgreSQL connection was established diff --git a/tests/unit/coal/test_postgresql/test_postgresql_store.py b/tests/unit/coal/test_postgresql/test_postgresql_store.py index 76e0c63a..e0cb824a 100644 --- a/tests/unit/coal/test_postgresql/test_postgresql_store.py +++ b/tests/unit/coal/test_postgresql/test_postgresql_store.py @@ -86,6 +86,7 @@ def test_dump_store_to_postgresql_with_tables(self, mock_send_to_postgresql, moc postgres_user, postgres_password, replace, + False, ), call( table2_data, @@ -97,6 +98,7 @@ def test_dump_store_to_postgresql_with_tables(self, mock_send_to_postgresql, moc postgres_user, postgres_password, replace, + False, ), ] ) @@ -235,4 +237,5 @@ def test_dump_store_to_postgresql_default_parameters(self, mock_send_to_postgres postgres_user, postgres_password, True, # Default replace is True + False, )