Config settings using env variables

README.md

@@ -81,10 +81,25 @@ docker run -d \
   --config /app/config.yaml run_all
 ```
 
+Or with environment variables for credentials:
+
+```bash
+docker run -d \
+  -v /path/to/your/config.yaml:/app/config.yaml \
+  -v /path/to/your/data:/app/data \
+  -e MYSQL_USER=root \
+  -e MYSQL_PASSWORD=secret \
+  -e CLICKHOUSE_USER=default \
+  -e CLICKHOUSE_PASSWORD=secret \
+  fippo/mysql-ch-replicator:latest \
+  --config /app/config.yaml run_all
+```
+
 Make sure to:
 1. Mount your configuration file using the `-v` flag
 2. Mount a persistent volume for the data directory
 3. Adjust the paths according to your setup
+4. Optionally use `-e` flags to override credentials via environment variables
 
 ## Usage
 
@@ -187,7 +202,7 @@ __Hint__: _set `initial_replication_threads` to a number of cpu cores to acceler
 
 ### Configuration
 
-`mysql_ch_replicator` can be configured through a configuration file. Here is the config example:
+`mysql_ch_replicator` can be configured through a configuration file and also by using enviromnent variables to override some of config settings. Here is the config example:
 
 ```yaml
 mysql:
@@ -299,6 +314,10 @@ databases: ['my_database_1', 'my_database_2']
 tables: ['table_1', 'table_2*']
 ```
 
+**Environment Variables**: MySQL and ClickHouse credentials can be overridden using environment variables for better security in containerized environments:
+- MySQL: `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_CHARSET`
+- ClickHouse: `CLICKHOUSE_HOST`, `CLICKHOUSE_PORT`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD`
+
 ### Advanced Features
 
 #### Migrations & Schema Changes

example_config.yaml

@@ -1,4 +1,8 @@
 
+# MySQL and ClickHouse credentials can be overridden using environment variables:
+# MySQL: MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD, MYSQL_CHARSET
+# ClickHouse: CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD
+
 mysql:
   host: 'localhost'
   port: 8306

mysql_ch_replicator/config.py

@@ -1,3 +1,4 @@
+import os
 import yaml
 import fnmatch
 import zoneinfo
@@ -157,8 +158,11 @@ def load(self, settings_file):
         data = yaml.safe_load(data)
 
         self.settings_file = settings_file
-        self.mysql = MysqlSettings(**data.pop('mysql'))
-        self.clickhouse = ClickhouseSettings(**data.pop('clickhouse'))
+        self.mysql = MysqlSettings(**data.pop('mysql', {}))
+        self.clickhouse = ClickhouseSettings(**data.pop('clickhouse', {}))
+
+        self._apply_env_overrides()
+
         self.databases = data.pop('databases')
         self.tables = data.pop('tables', '*')
         self.exclude_databases = data.pop('exclude_databases', '')
@@ -206,6 +210,27 @@ def load(self, settings_file):
             raise Exception(f'Unsupported config options: {list(data.keys())}')
         self.validate()
 
+    def _apply_env_overrides(self):
+        if os.getenv('MYSQL_HOST'):
+            self.mysql.host = os.getenv('MYSQL_HOST')
+        if os.getenv('MYSQL_PORT'):
+            self.mysql.port = int(os.getenv('MYSQL_PORT'))
+        if os.getenv('MYSQL_USER'):
+            self.mysql.user = os.getenv('MYSQL_USER')
+        if os.getenv('MYSQL_PASSWORD'):
+            self.mysql.password = os.getenv('MYSQL_PASSWORD')
+        if os.getenv('MYSQL_CHARSET'):
+            self.mysql.charset = os.getenv('MYSQL_CHARSET')
+
+        if os.getenv('CLICKHOUSE_HOST'):
+            self.clickhouse.host = os.getenv('CLICKHOUSE_HOST')
+        if os.getenv('CLICKHOUSE_PORT'):
+            self.clickhouse.port = int(os.getenv('CLICKHOUSE_PORT'))
+        if os.getenv('CLICKHOUSE_USER'):
+            self.clickhouse.user = os.getenv('CLICKHOUSE_USER')
+        if os.getenv('CLICKHOUSE_PASSWORD'):
+            self.clickhouse.password = os.getenv('CLICKHOUSE_PASSWORD')
+
     @classmethod
     def is_pattern_matches(cls, substr, pattern):
         if not pattern or pattern == '*':

tests/test_config_env_vars.py

@@ -0,0 +1,122 @@
+import os
+from pathlib import Path
+
+from mysql_ch_replicator.config import Settings
+
+
+def test_env_vars_override_config():
+    config_file = Path(__file__).parent / 'tests_config_env_vars.yaml'
+
+    os.environ['MYSQL_HOST'] = 'mysql.env.host'
+    os.environ['MYSQL_PORT'] = '8306'
+    os.environ['MYSQL_USER'] = 'env_mysql_user'
+    os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass'
+    os.environ['MYSQL_CHARSET'] = 'utf8'
+
+    os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host'
+    os.environ['CLICKHOUSE_PORT'] = '8323'
+    os.environ['CLICKHOUSE_USER'] = 'env_ch_user'
+    os.environ['CLICKHOUSE_PASSWORD'] = 'env_ch_pass'
+
+    settings = Settings()
+    settings.load(str(config_file))
+
+    assert settings.mysql.host == 'mysql.env.host'
+    assert settings.mysql.port == 8306
+    assert settings.mysql.user == 'env_mysql_user'
+    assert settings.mysql.password == 'env_mysql_pass'
+    assert settings.mysql.charset == 'utf8'
+
+    assert settings.clickhouse.host == 'clickhouse.env.host'
+    assert settings.clickhouse.port == 8323
+    assert settings.clickhouse.user == 'env_ch_user'
+    assert settings.clickhouse.password == 'env_ch_pass'
+
+    del os.environ['MYSQL_HOST']
+    del os.environ['MYSQL_PORT']
+    del os.environ['MYSQL_USER']
+    del os.environ['MYSQL_PASSWORD']
+    del os.environ['MYSQL_CHARSET']
+    del os.environ['CLICKHOUSE_HOST']
+    del os.environ['CLICKHOUSE_PORT']
+    del os.environ['CLICKHOUSE_USER']
+    del os.environ['CLICKHOUSE_PASSWORD']
+
+
+def test_config_without_env_vars():
+    config_file = Path(__file__).parent / 'tests_config_env_vars.yaml'
+
+    settings = Settings()
+    settings.load(str(config_file))
+
+    assert settings.mysql.host == 'mysql.local'
+    assert settings.mysql.port == 3306
+    assert settings.mysql.user == 'mysql_user'
+    assert settings.mysql.password == 'mysql_pass'
+    assert settings.mysql.charset == 'utf8mb4'
+
+    assert settings.clickhouse.host == 'clickhouse.local'
+    assert settings.clickhouse.port == 9000
+    assert settings.clickhouse.user == 'ch_user'
+    assert settings.clickhouse.password == 'ch_pass'
+
+
+def test_partial_env_vars_override():
+    config_file = Path(__file__).parent / 'tests_config_env_vars.yaml'
+
+    os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass'
+    os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host'
+
+    settings = Settings()
+    settings.load(str(config_file))
+
+    assert settings.mysql.host == 'mysql.local'
+    assert settings.mysql.port == 3306
+    assert settings.mysql.user == 'mysql_user'
+    assert settings.mysql.password == 'env_mysql_pass'
+    assert settings.mysql.charset == 'utf8mb4'
+
+    assert settings.clickhouse.host == 'clickhouse.env.host'
+    assert settings.clickhouse.port == 9000
+    assert settings.clickhouse.user == 'ch_user'
+    assert settings.clickhouse.password == 'ch_pass'
+
+    del os.environ['MYSQL_PASSWORD']
+    del os.environ['CLICKHOUSE_HOST']
+
+
+def test_config_without_mysql_clickhouse_sections():
+    config_file = Path(__file__).parent / 'tests_config_env_vars_no_creds.yaml'
+
+    os.environ['MYSQL_HOST'] = 'mysql.env.host'
+    os.environ['MYSQL_PORT'] = '8306'
+    os.environ['MYSQL_USER'] = 'env_mysql_user'
+    os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass'
+
+    os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host'
+    os.environ['CLICKHOUSE_PORT'] = '8323'
+    os.environ['CLICKHOUSE_USER'] = 'env_ch_user'
+    os.environ['CLICKHOUSE_PASSWORD'] = 'env_ch_pass'
+
+    settings = Settings()
+    settings.load(str(config_file))
+
+    assert settings.mysql.host == 'mysql.env.host'
+    assert settings.mysql.port == 8306
+    assert settings.mysql.user == 'env_mysql_user'
+    assert settings.mysql.password == 'env_mysql_pass'
+
+    assert settings.clickhouse.host == 'clickhouse.env.host'
+    assert settings.clickhouse.port == 8323
+    assert settings.clickhouse.user == 'env_ch_user'
+    assert settings.clickhouse.password == 'env_ch_pass'
+
+    del os.environ['MYSQL_HOST']
+    del os.environ['MYSQL_PORT']
+    del os.environ['MYSQL_USER']
+    del os.environ['MYSQL_PASSWORD']
+    del os.environ['CLICKHOUSE_HOST']
+    del os.environ['CLICKHOUSE_PORT']
+    del os.environ['CLICKHOUSE_USER']
+    del os.environ['CLICKHOUSE_PASSWORD']
+

tests/tests_config_env_vars.yaml

@@ -0,0 +1,20 @@
+mysql:
+  host: 'mysql.local'
+  port: 3306
+  user: 'mysql_user'
+  password: 'mysql_pass'
+  charset: 'utf8mb4'
+
+clickhouse:
+  host: 'clickhouse.local'
+  port: 9000
+  user: 'ch_user'
+  password: 'ch_pass'
+
+binlog_replicator:
+  data_dir: '/tmp/binlog'
+  records_per_file: 100000
+
+databases: 'test_db'
+log_level: 'info'
+

tests/tests_config_env_vars_no_creds.yaml

@@ -0,0 +1,7 @@
+binlog_replicator:
+  data_dir: '/tmp/binlog'
+  records_per_file: 100000
+
+databases: 'test_db'
+log_level: 'info'
+