Merge pull request #5 from d-ganchar/save_db_structure

save_db_structure

Author: d-ganchar

.github/workflows/ci_cd.yml

@@ -48,22 +48,6 @@ jobs:
     runs-on: ubuntu-22.04
     permissions: write-all
     needs: [ build ]
-
-    services:
-      clickhouse:
-        image: clickhouse/clickhouse-server:23.4
-        env:
-          CLICKHOUSE_USER: thedus_tests
-          CLICKHOUSE_PASSWORD: thedus_tests
-        ports:
-          - 9000:9000
-          - 8123:8123
-        options: >-
-          --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
     strategy:
       fail-fast: false
       matrix:
@@ -78,6 +62,17 @@ jobs:
       - name: Install dependencies
         run: uv sync --dev
 
+      - uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          services: |
+            clickhouse
+          up-flags: "-d"
+
+      - uses: iFaxity/wait-on-action@v1.2.1
+        with:
+          resource: http://localhost:8123
+          timeout: 30000
+
       - name: Download thedus dist
         uses: actions/download-artifact@v4
         with:

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:23.4.2.11-alpine
+    image: clickhouse/clickhouse-server:25.4.2.31
     container_name: thedus_clickhouse
     logging:
       driver: none
@@ -15,5 +15,6 @@ services:
       timeout: 2s
       retries: 16
     environment:
+      CLICKHOUSE_DB: thedus_tests
       CLICKHOUSE_USER: thedus_tests
       CLICKHOUSE_PASSWORD: thedus_tests

pyproject.toml

@@ -9,9 +9,10 @@ readme = "README.md"
 requires-python = ">=3.9, < 3.14"
 dependencies = [
   "rich>=13.9, < 14",
-  "typer>=0.15, < 0.16",
-  "clickhouse-driver>=0.2.9, < 0.3",
-  "ripley==0.1.0b0"
+  "typer>=0.15, < 1",
+  "clickhouse-driver>=0.2.9, < 1",
+  "ripley==0.3.0a0",
+  "textual>=3.1.1, < 4.0",
 ]
 
 classifiers = [

tests/test_cli.py

@@ -2,40 +2,39 @@
 import os
 import subprocess
 import unittest
-
-import platform
 from typing import List
 
-from parameterized import parameterized
 from clickhouse_driver import Client
+from ripley import from_clickhouse
+from parameterized import parameterized
 
 
-os.environ['CLICKHOUSE_USER'] = 'thedus_tests'
-os.environ['CLICKHOUSE_PASSWORD'] = 'thedus_tests'
+_CLICKHOUSE_DB = 'thedus_tests'
+_THEDUS_DIR = os.path.join(os.path.dirname(__file__), 'migrations')
 
-
-def init_clickhouse(database: str = 'default') -> Client:
-    return Client(
-        host='localhost',
-        port=9000,
-        user=os.environ['CLICKHOUSE_USER'],
-        password=os.environ['CLICKHOUSE_PASSWORD'],
-        database=database,
-    )
+os.environ['THEDUS_DIR'] = _THEDUS_DIR
+os.environ['CLICKHOUSE_DB'] = _CLICKHOUSE_DB
+os.environ['CLICKHOUSE_PASSWORD'] = _CLICKHOUSE_DB
+os.environ['CLICKHOUSE_USER'] = _CLICKHOUSE_DB
 
 
 class BaseCliTest(unittest.TestCase):
     maxDiff = 10000
-    clickhouse = init_clickhouse()
+    clickhouse = Client(
+        host='localhost',
+        port=9000,
+        user=_CLICKHOUSE_DB,
+        password=_CLICKHOUSE_DB,
+        database=_CLICKHOUSE_DB,
+    )
 
     @property
     def db_name(self):
-        minor, major, _ = platform.python_version().split('.')
-        return 'thedus_' + '_'.join([minor, major])
+        return _CLICKHOUSE_DB
 
     @property
     def thedus_dir(self) -> str:
-        return os.environ['THEDUS_DIR']
+        return _THEDUS_DIR
 
     @property
     def test_tables(self) -> List[str]:
@@ -46,21 +45,17 @@ def test_tables(self) -> List[str]:
         ]
 
     def setUp(self):
-        thedus_dir = os.path.join(os.path.dirname(__file__), 'migrations')
         os.environ['THEDUS_ENV'] = ''
-        os.environ['THEDUS_DIR'] = thedus_dir
-        os.environ['CLICKHOUSE_DB'] = self.db_name
 
-        with os.scandir(thedus_dir) as entries:
+        with os.scandir(self.thedus_dir) as entries:
             for entry in entries:
                 if entry.name == '.gitignore':
                     continue
                 os.remove(entry.path)
 
-        self.clickhouse = init_clickhouse()
-        self.clickhouse.execute(f'DROP DATABASE IF EXISTS {self.db_name}')
-        self.clickhouse.execute(f'CREATE DATABASE IF NOT EXISTS {self.db_name}')
-        self.clickhouse = init_clickhouse(self.db_name)
+        clickhouse = from_clickhouse(self.clickhouse)
+        for table in clickhouse.get_tables_by_db(_CLICKHOUSE_DB):
+            self.clickhouse.execute(f"DROP TABLE {table.full_name}")
 
         for file_name, up, down, skip_env in (
             (
@@ -269,3 +264,83 @@ def test_downgrade_to_revision(self):
         self.check_thedus_output(result, ['rollback 20250101000000_0_create_tbl_metrics', 'done'])
         result = subprocess.getoutput('thedus downgrade')
         self.check_thedus_output(result, ['done'])
+
+
+class TestSaveDbStructure(BaseCliTest):
+    def test_save_db_structure(self):
+        self.clickhouse.execute("""
+            CREATE TABLE votes
+            (
+                `Id` UInt32,
+                `PostId` Int32,
+                `VoteTypeId` UInt8,
+                `CreationDate` DateTime64(3, 'UTC'),
+                `UserId` Int32,
+                `BountyAmount` UInt8
+            )
+            ENGINE = MergeTree
+            ORDER BY (VoteTypeId, CreationDate, PostId)
+        """)
+
+        self.clickhouse.execute("""
+            CREATE TABLE up_down_votes_per_day
+            (
+              `Day` Date,
+              `UpVotes` UInt32,
+              `DownVotes` UInt32
+            )
+            ENGINE = SummingMergeTree
+            ORDER BY Day
+        """)
+
+        self.clickhouse.execute("""
+            CREATE MATERIALIZED VIEW up_down_votes_per_day_mv TO up_down_votes_per_day AS
+            SELECT toStartOfDay(CreationDate)::Date AS Day,
+                   countIf(VoteTypeId = 2) AS UpVotes,
+                   countIf(VoteTypeId = 3) AS DownVotes
+              FROM votes
+             GROUP BY Day
+        """)
+
+        self.clickhouse.execute("""
+            CREATE VIEW upvotes_per_user AS
+            SELECT toDate(CreationDate) AS Day,
+                   UserId,
+                   count() AS user_votes
+              FROM votes
+             GROUP BY Day, UserId
+        """)
+
+        result = subprocess.run(
+            'thedus save-db-structure',
+            shell=True,
+            cwd=self.thedus_dir,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            env=os.environ,
+        )
+
+        filename = f'{_CLICKHOUSE_DB}.sql'
+        self.assertEqual(result.stdout[32:], f'./{filename} created\n')
+
+        with codecs.open(os.path.join(self.thedus_dir, filename)) as file:
+            self.assertEqual(
+                (
+                    f"CREATE TABLE {_CLICKHOUSE_DB}.up_down_votes_per_day "
+                    "(`Day` Date, `UpVotes` UInt32, `DownVotes` UInt32) "
+                    "ENGINE = SummingMergeTree ORDER BY Day SETTINGS index_granularity = 8192;\n"
+                    f"CREATE TABLE {_CLICKHOUSE_DB}.votes (`Id` UInt32, `PostId` Int32, `VoteTypeId` UInt8, "
+                    "`CreationDate` DateTime64(3, 'UTC'), `UserId` Int32, `BountyAmount` UInt8) ENGINE = MergeTree "
+                    "ORDER BY (VoteTypeId, CreationDate, PostId) SETTINGS index_granularity = 8192;\n"
+                    "CREATE MATERIALIZED VIEW "
+                    f"{_CLICKHOUSE_DB}.up_down_votes_per_day_mv TO {_CLICKHOUSE_DB}.up_down_votes_per_day "
+                    "(`Day` Date, `UpVotes` UInt64, `DownVotes` UInt64) AS "
+                    "SELECT CAST(toStartOfDay(CreationDate), 'Date') AS Day, countIf(VoteTypeId = 2) AS UpVotes, "
+                    f"countIf(VoteTypeId = 3) AS DownVotes FROM {_CLICKHOUSE_DB}.votes GROUP BY Day;\n"
+                    f"CREATE VIEW {_CLICKHOUSE_DB}.upvotes_per_user (`Day` Date, `UserId` Int32, `user_votes` UInt64) "
+                    f"AS SELECT toDate(CreationDate) AS Day, UserId, count() AS user_votes FROM {_CLICKHOUSE_DB}.votes "
+                    "GROUP BY Day, UserId"
+                ),
+                file.read(),
+            )

thedus/run.py

@@ -1,5 +1,6 @@
-import os
+import codecs
 import logging
+import os
 from datetime import timezone, datetime
 from logging.config import dictConfig
 
@@ -9,10 +10,9 @@
 from ripley import ClickhouseProtocol, from_clickhouse
 from typing_extensions import Annotated
 
+from .cmd.clickhouse_cmd import StateCmd, UpgradeCmd, DowngradeCmd
 from .env_manager import EnvManager as Env
 from .migration_file_manager import MigrationFileManager as Migrations
-from .cmd.clickhouse_cmd import StateCmd, UpgradeCmd, DowngradeCmd
-
 
 dictConfig({
     'version': 1,
@@ -40,7 +40,7 @@
 })
 
 
-def init_clickhouse() -> ClickhouseProtocol:
+def init_clickhouse(create_migration_log: bool=True) -> ClickhouseProtocol:
     host = Env.get_clickhouse_host()
     db = Env.get_clickhouse_db()
 
@@ -53,17 +53,19 @@ def init_clickhouse() -> ClickhouseProtocol:
             database=db,
         ))
 
-        clickhouse.exec("""CREATE TABLE IF NOT EXISTS thedus_migration_log
-        (
-            command String,
-            revision String,
-            environment String,
-            version UInt64,
-            is_skipped UInt8 default 0,
-            created_at Datetime default now()
-        )
-        ENGINE = Log
-        """)
+        if create_migration_log:
+            clickhouse.exec("""
+            CREATE TABLE IF NOT EXISTS thedus_migration_log
+            (
+                command String,
+                revision String,
+                environment String,
+                version UInt64,
+                is_skipped UInt8 default 0,
+                created_at Datetime default now()
+            )
+            ENGINE = Log
+            """)
 
         return clickhouse
     except (Error, ConnectionRefusedError) as error:
@@ -168,5 +170,43 @@ def downgrade(
     logging.info('done')
 
 
+@app.command(help='Saves a database structure to a file, system.tables.create_table_query is used to generate the DDL')
+def save_db_structure(
+    db_name: Annotated[
+        str,
+        typer.Argument(
+            metavar='TEXT',
+            help='Database name'
+        ),
+    ] = Env.get_clickhouse_db(),
+    file_path: Annotated[
+        str,
+        typer.Argument(
+            metavar='TEXT',
+            help='Full path to output file. default: "./$CLICKHOUSE_DB.sql"'
+        ),
+    ] = f'./{Env.get_clickhouse_db()}.sql',
+):
+    if os.path.exists(file_path):
+        logging.error(f'File {file_path} already exists')
+        exit(1)
+
+    clickhouse = init_clickhouse(False)
+    tables = clickhouse.get_tables_by_db(db_name)
+    tables = sorted(
+        tables,
+        key=lambda t: (t.engine.lower() in ('view', 'materializedview'), t.metadata_modification_time)
+    )
+
+    if not tables:
+        logging.warning('"%s" objects not found', db_name)
+        exit(0)
+
+    with codecs.open(file_path, 'w') as file:
+        file.write(';\n'.join([t.create_table_query for t in tables]))
+
+    logging.info('%s created', file_path)
+
+
 if __name__ == '__main__':
     app()