Skip to content

Commit 8de957c

Browse files
bakwcbakwc
authored
Config settings using env variables (#213)
1 parent 67598cf commit 8de957c

File tree

6 files changed

+200
-3
lines changed

6 files changed

+200
-3
lines changed

‎README.md‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,25 @@ docker run -d \
8181
--config /app/config.yaml run_all
8282
```
8383

84+
Or with environment variables for credentials:
85+
86+
```bash
87+
docker run -d \
88+
-v /path/to/your/config.yaml:/app/config.yaml \
89+
-v /path/to/your/data:/app/data \
90+
-e MYSQL_USER=root \
91+
-e MYSQL_PASSWORD=secret \
92+
-e CLICKHOUSE_USER=default \
93+
-e CLICKHOUSE_PASSWORD=secret \
94+
fippo/mysql-ch-replicator:latest \
95+
--config /app/config.yaml run_all
96+
```
97+
8498
Make sure to:
8599
1. Mount your configuration file using the `-v` flag
86100
2. Mount a persistent volume for the data directory
87101
3. Adjust the paths according to your setup
102+
4. Optionally use `-e` flags to override credentials via environment variables
88103

89104
## Usage
90105

@@ -187,7 +202,7 @@ __Hint__: _set `initial_replication_threads` to a number of cpu cores to acceler
187202

188203
### Configuration
189204

190-
`mysql_ch_replicator` can be configured through a configuration file. Here is the config example:
205+
`mysql_ch_replicator` can be configured through a configuration file and also by using enviromnent variables to override some of config settings. Here is the config example:
191206

192207
```yaml
193208
mysql:
@@ -299,6 +314,10 @@ databases: ['my_database_1', 'my_database_2']
299314
tables: ['table_1', 'table_2*']
300315
```
301316

317+
**Environment Variables**: MySQL and ClickHouse credentials can be overridden using environment variables for better security in containerized environments:
318+
- MySQL: `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_CHARSET`
319+
- ClickHouse: `CLICKHOUSE_HOST`, `CLICKHOUSE_PORT`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD`
320+
302321
### Advanced Features
303322

304323
#### Migrations & Schema Changes

‎example_config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11

2+
# MySQL and ClickHouse credentials can be overridden using environment variables:
3+
# MySQL: MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD, MYSQL_CHARSET
4+
# ClickHouse: CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD
5+
26
mysql:
37
host: 'localhost'
48
port: 8306

‎mysql_ch_replicator/config.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
import yaml
23
import fnmatch
34
import zoneinfo
@@ -157,8 +158,11 @@ def load(self, settings_file):
157158
data = yaml.safe_load(data)
158159

159160
self.settings_file = settings_file
160-
self.mysql = MysqlSettings(**data.pop('mysql'))
161-
self.clickhouse = ClickhouseSettings(**data.pop('clickhouse'))
161+
self.mysql = MysqlSettings(**data.pop('mysql', {}))
162+
self.clickhouse = ClickhouseSettings(**data.pop('clickhouse', {}))
163+
164+
self._apply_env_overrides()
165+
162166
self.databases = data.pop('databases')
163167
self.tables = data.pop('tables', '*')
164168
self.exclude_databases = data.pop('exclude_databases', '')
@@ -206,6 +210,27 @@ def load(self, settings_file):
206210
raise Exception(f'Unsupported config options: {list(data.keys())}')
207211
self.validate()
208212

213+
def _apply_env_overrides(self):
214+
if os.getenv('MYSQL_HOST'):
215+
self.mysql.host = os.getenv('MYSQL_HOST')
216+
if os.getenv('MYSQL_PORT'):
217+
self.mysql.port = int(os.getenv('MYSQL_PORT'))
218+
if os.getenv('MYSQL_USER'):
219+
self.mysql.user = os.getenv('MYSQL_USER')
220+
if os.getenv('MYSQL_PASSWORD'):
221+
self.mysql.password = os.getenv('MYSQL_PASSWORD')
222+
if os.getenv('MYSQL_CHARSET'):
223+
self.mysql.charset = os.getenv('MYSQL_CHARSET')
224+
225+
if os.getenv('CLICKHOUSE_HOST'):
226+
self.clickhouse.host = os.getenv('CLICKHOUSE_HOST')
227+
if os.getenv('CLICKHOUSE_PORT'):
228+
self.clickhouse.port = int(os.getenv('CLICKHOUSE_PORT'))
229+
if os.getenv('CLICKHOUSE_USER'):
230+
self.clickhouse.user = os.getenv('CLICKHOUSE_USER')
231+
if os.getenv('CLICKHOUSE_PASSWORD'):
232+
self.clickhouse.password = os.getenv('CLICKHOUSE_PASSWORD')
233+
209234
@classmethod
210235
def is_pattern_matches(cls, substr, pattern):
211236
if not pattern or pattern == '*':

‎tests/test_config_env_vars.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import os
2+
from pathlib import Path
3+
4+
from mysql_ch_replicator.config import Settings
5+
6+
7+
def test_env_vars_override_config():
8+
config_file = Path(__file__).parent / 'tests_config_env_vars.yaml'
9+
10+
os.environ['MYSQL_HOST'] = 'mysql.env.host'
11+
os.environ['MYSQL_PORT'] = '8306'
12+
os.environ['MYSQL_USER'] = 'env_mysql_user'
13+
os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass'
14+
os.environ['MYSQL_CHARSET'] = 'utf8'
15+
16+
os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host'
17+
os.environ['CLICKHOUSE_PORT'] = '8323'
18+
os.environ['CLICKHOUSE_USER'] = 'env_ch_user'
19+
os.environ['CLICKHOUSE_PASSWORD'] = 'env_ch_pass'
20+
21+
settings = Settings()
22+
settings.load(str(config_file))
23+
24+
assert settings.mysql.host == 'mysql.env.host'
25+
assert settings.mysql.port == 8306
26+
assert settings.mysql.user == 'env_mysql_user'
27+
assert settings.mysql.password == 'env_mysql_pass'
28+
assert settings.mysql.charset == 'utf8'
29+
30+
assert settings.clickhouse.host == 'clickhouse.env.host'
31+
assert settings.clickhouse.port == 8323
32+
assert settings.clickhouse.user == 'env_ch_user'
33+
assert settings.clickhouse.password == 'env_ch_pass'
34+
35+
del os.environ['MYSQL_HOST']
36+
del os.environ['MYSQL_PORT']
37+
del os.environ['MYSQL_USER']
38+
del os.environ['MYSQL_PASSWORD']
39+
del os.environ['MYSQL_CHARSET']
40+
del os.environ['CLICKHOUSE_HOST']
41+
del os.environ['CLICKHOUSE_PORT']
42+
del os.environ['CLICKHOUSE_USER']
43+
del os.environ['CLICKHOUSE_PASSWORD']
44+
45+
46+
def test_config_without_env_vars():
47+
config_file = Path(__file__).parent / 'tests_config_env_vars.yaml'
48+
49+
settings = Settings()
50+
settings.load(str(config_file))
51+
52+
assert settings.mysql.host == 'mysql.local'
53+
assert settings.mysql.port == 3306
54+
assert settings.mysql.user == 'mysql_user'
55+
assert settings.mysql.password == 'mysql_pass'
56+
assert settings.mysql.charset == 'utf8mb4'
57+
58+
assert settings.clickhouse.host == 'clickhouse.local'
59+
assert settings.clickhouse.port == 9000
60+
assert settings.clickhouse.user == 'ch_user'
61+
assert settings.clickhouse.password == 'ch_pass'
62+
63+
64+
def test_partial_env_vars_override():
65+
config_file = Path(__file__).parent / 'tests_config_env_vars.yaml'
66+
67+
os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass'
68+
os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host'
69+
70+
settings = Settings()
71+
settings.load(str(config_file))
72+
73+
assert settings.mysql.host == 'mysql.local'
74+
assert settings.mysql.port == 3306
75+
assert settings.mysql.user == 'mysql_user'
76+
assert settings.mysql.password == 'env_mysql_pass'
77+
assert settings.mysql.charset == 'utf8mb4'
78+
79+
assert settings.clickhouse.host == 'clickhouse.env.host'
80+
assert settings.clickhouse.port == 9000
81+
assert settings.clickhouse.user == 'ch_user'
82+
assert settings.clickhouse.password == 'ch_pass'
83+
84+
del os.environ['MYSQL_PASSWORD']
85+
del os.environ['CLICKHOUSE_HOST']
86+
87+
88+
def test_config_without_mysql_clickhouse_sections():
89+
config_file = Path(__file__).parent / 'tests_config_env_vars_no_creds.yaml'
90+
91+
os.environ['MYSQL_HOST'] = 'mysql.env.host'
92+
os.environ['MYSQL_PORT'] = '8306'
93+
os.environ['MYSQL_USER'] = 'env_mysql_user'
94+
os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass'
95+
96+
os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host'
97+
os.environ['CLICKHOUSE_PORT'] = '8323'
98+
os.environ['CLICKHOUSE_USER'] = 'env_ch_user'
99+
os.environ['CLICKHOUSE_PASSWORD'] = 'env_ch_pass'
100+
101+
settings = Settings()
102+
settings.load(str(config_file))
103+
104+
assert settings.mysql.host == 'mysql.env.host'
105+
assert settings.mysql.port == 8306
106+
assert settings.mysql.user == 'env_mysql_user'
107+
assert settings.mysql.password == 'env_mysql_pass'
108+
109+
assert settings.clickhouse.host == 'clickhouse.env.host'
110+
assert settings.clickhouse.port == 8323
111+
assert settings.clickhouse.user == 'env_ch_user'
112+
assert settings.clickhouse.password == 'env_ch_pass'
113+
114+
del os.environ['MYSQL_HOST']
115+
del os.environ['MYSQL_PORT']
116+
del os.environ['MYSQL_USER']
117+
del os.environ['MYSQL_PASSWORD']
118+
del os.environ['CLICKHOUSE_HOST']
119+
del os.environ['CLICKHOUSE_PORT']
120+
del os.environ['CLICKHOUSE_USER']
121+
del os.environ['CLICKHOUSE_PASSWORD']
122+

‎tests/tests_config_env_vars.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
mysql:
2+
host: 'mysql.local'
3+
port: 3306
4+
user: 'mysql_user'
5+
password: 'mysql_pass'
6+
charset: 'utf8mb4'
7+
8+
clickhouse:
9+
host: 'clickhouse.local'
10+
port: 9000
11+
user: 'ch_user'
12+
password: 'ch_pass'
13+
14+
binlog_replicator:
15+
data_dir: '/tmp/binlog'
16+
records_per_file: 100000
17+
18+
databases: 'test_db'
19+
log_level: 'info'
20+

‎tests/tests_config_env_vars_no_creds.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
binlog_replicator:
2+
data_dir: '/tmp/binlog'
3+
records_per_file: 100000
4+
5+
databases: 'test_db'
6+
log_level: 'info'
7+

0 commit comments

Comments
 (0)