Tests separation (#201)

Author: bakwc

.github/workflows/tests.yaml

@@ -19,4 +19,4 @@ jobs:
           cd tests &&
           ls -la &&
           docker compose -f docker-compose-tests.yaml up --force-recreate --no-deps --wait -d &&
-          sudo docker exec -w /app/ -i `docker ps | grep tests-replicator | awk '{print $1;}'` python3 -m pytest -x -v -s tests/test_mysql_ch_replicator.py
+          sudo docker exec -w /app/ -i `docker ps | grep tests-replicator | awk '{print $1;}'` python3 -m pytest -x -v -s tests/

README.md

@@ -336,11 +336,11 @@ docker compose -f docker-compose-tests.yaml up
 ```
 2. Run tests with:
 ```bash
-docker exec -w /app/ -it tests-replicator-1 python3 -m pytest -v -s tests/test_mysql_ch_replicator.py
+docker exec -w /app/ -it tests-replicator-1 python3 -m pytest -v -s tests/
 ```
 3. To run a single test:
 ```bash
-docker exec -w /app/ -it tests-replicator-1 python3 -m pytest -v -s tests/test_mysql_ch_replicator.py -k test_your_test_name
+docker exec -w /app/ -it tests-replicator-1 python3 -m pytest -v -s tests/ -k test_your_test_name
 ```
 
 ## Contribution

tests/common.py

@@ -0,0 +1,99 @@
+import os
+import shutil
+import subprocess
+import time
+
+import requests
+
+from mysql_ch_replicator import clickhouse_api
+from mysql_ch_replicator import config
+from mysql_ch_replicator import mysql_api
+from mysql_ch_replicator.binlog_replicator import State as BinlogState
+from mysql_ch_replicator.db_replicator import State as DbReplicatorState
+from mysql_ch_replicator.runner import ProcessRunner
+
+
+CONFIG_FILE = 'tests/tests_config.yaml'
+CONFIG_FILE_MARIADB = 'tests/tests_config_mariadb.yaml'
+TEST_DB_NAME = 'replication-test_db'
+TEST_DB_NAME_2 = 'replication-test_db_2'
+TEST_DB_NAME_2_DESTINATION = 'replication-destination'
+
+TEST_TABLE_NAME = 'test_table'
+TEST_TABLE_NAME_2 = 'test_table_2'
+TEST_TABLE_NAME_3 = 'test_table_3'
+
+
+class BinlogReplicatorRunner(ProcessRunner):
+    def __init__(self, cfg_file=CONFIG_FILE):
+        super().__init__(f'./main.py --config {cfg_file} binlog_replicator')
+
+
+class DbReplicatorRunner(ProcessRunner):
+    def __init__(self, db_name, additional_arguments=None, cfg_file=CONFIG_FILE):
+        additional_arguments = additional_arguments or ''
+        if not additional_arguments.startswith(' '):
+            additional_arguments = ' ' + additional_arguments
+        super().__init__(f'./main.py --config {cfg_file} --db {db_name} db_replicator{additional_arguments}')
+
+
+class RunAllRunner(ProcessRunner):
+    def __init__(self, cfg_file=CONFIG_FILE):
+        super().__init__(f'./main.py --config {cfg_file} run_all')
+
+
+def kill_process(pid, force=False):
+    command = f'kill {pid}'
+    if force:
+        command = f'kill -9 {pid}'
+    subprocess.run(command, shell=True)
+
+
+def assert_wait(condition, max_wait_time=20.0, retry_interval=0.05):
+    max_time = time.time() + max_wait_time
+    while time.time() < max_time:
+        if condition():
+            return
+        time.sleep(retry_interval)
+    assert condition()
+
+
+def prepare_env(
+        cfg: config.Settings,
+        mysql: mysql_api.MySQLApi,
+        ch: clickhouse_api.ClickhouseApi,
+        db_name: str = TEST_DB_NAME,
+        set_mysql_db: bool = True
+):
+    if os.path.exists(cfg.binlog_replicator.data_dir):
+        shutil.rmtree(cfg.binlog_replicator.data_dir)
+    os.mkdir(cfg.binlog_replicator.data_dir)
+    mysql.drop_database(db_name)
+    mysql.create_database(db_name)
+    if set_mysql_db:
+        mysql.set_database(db_name)
+    ch.drop_database(db_name)
+    assert_wait(lambda: db_name not in ch.get_databases())
+
+
+def get_binlog_replicator_pid(cfg: config.Settings):
+    path = os.path.join(
+        cfg.binlog_replicator.data_dir,
+        'state.json',
+    )
+    state = BinlogState(path)
+    return state.pid
+
+
+def get_db_replicator_pid(cfg: config.Settings, db_name: str):
+    path = os.path.join(
+        cfg.binlog_replicator.data_dir,
+        db_name,
+        'state.pckl',
+    )
+    state = DbReplicatorState(path)
+    return state.pid
+
+
+def read_logs(db_name):
+    return open(os.path.join('binlog', db_name, 'db_replicator.log')).read()