Merge pull request #3 from d-ganchar/migration_file_name

added numbers to revision name, reg exp: ^[0-9]{14}[a-z0-9_]+.py$

Author: d-ganchar

.github/workflows/ci_cd.yml

@@ -11,6 +11,7 @@ permissions:
   pages: write
   id-token: write
 
+
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -47,11 +48,16 @@ jobs:
     runs-on: ubuntu-22.04
     permissions: write-all
     needs: [ build ]
+
     services:
       clickhouse:
-        image: clickhouse/clickhouse-server:latest
+        image: clickhouse/clickhouse-server:23.4
+        env:
+          CLICKHOUSE_USER: thedus_tests
+          CLICKHOUSE_PASSWORD: thedus_tests
         ports:
           - 9000:9000
+          - 8123:8123
         options: >-
           --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1"
           --health-interval 10s

docker-compose.yml

@@ -14,3 +14,6 @@ services:
       interval: 2s
       timeout: 2s
       retries: 16
+    environment:
+      CLICKHOUSE_USER: thedus_tests
+      CLICKHOUSE_PASSWORD: thedus_tests

pyproject.toml

@@ -11,7 +11,7 @@ dependencies = [
   "rich>=13.9, < 14",
   "typer>=0.15, < 0.16",
   "clickhouse-driver>=0.2.9, < 0.3",
-  "ripley==0.2.0a0"
+  "ripley==0.1.0b0"
 ]
 
 classifiers = [
@@ -41,8 +41,8 @@ build-backend = "setuptools.build_meta"
 
 [dependency-groups]
 dev = [
-    "ruff>=0.9.1, <1",
-    "parameterized==0.9.0",
+    "ruff>=0.11.4, <1",
+    "parameterized==0.9.0, < 1",
     "pytest>=8.3.4, <9",
 ]
 

tests/migrations/.gitignore

@@ -10,12 +10,16 @@
 from clickhouse_driver import Client
 
 
+os.environ['CLICKHOUSE_USER'] = 'thedus_tests'
+os.environ['CLICKHOUSE_PASSWORD'] = 'thedus_tests'
+
+
 def init_clickhouse(database: str = 'default') -> Client:
     return Client(
         host='localhost',
         port=9000,
-        user='default',
-        password='',
+        user=os.environ['CLICKHOUSE_USER'],
+        password=os.environ['CLICKHOUSE_PASSWORD'],
         database=database,
     )
 
@@ -84,7 +88,7 @@ def setUp(self):
                 '',
             ),
         ):
-            file_path = os.path.join(self.thedus_dir, f'2025010100000{file_name}.py')
+            file_path = os.path.join(self.thedus_dir, f'20250101000000_{file_name}.py')
             get_env_to_skip = f"""
     @classmethod
     def get_env_to_skip(cls) -> list:
@@ -150,32 +154,32 @@ def down(self):
         [
             'dev',
             [
-                'upgrade to 20250101000000_create_tbl_metrics',
-                'SKIP 20250101000001_insert_into_metrics',
-                'upgrade to 20250101000002_create_tbl_events',
-                'upgrade to 20250101000003_create_tbl_logs',
+                'upgrade to 20250101000000_0_create_tbl_metrics',
+                'SKIP 20250101000000_1_insert_into_metrics',
+                'upgrade to 20250101000000_2_create_tbl_events',
+                'upgrade to 20250101000000_3_create_tbl_logs',
                 'done',
             ],
             [[(0,)], [(0,)], [(0,)]],
-            [('upgrade', '20250101000000_create_tbl_metrics', 'dev', 0),
-             ('upgrade', '20250101000001_insert_into_metrics', 'dev', 1),
-             ('upgrade', '20250101000002_create_tbl_events', 'dev', 0),
-             ('upgrade', '20250101000003_create_tbl_logs', 'dev', 0)],
+            [('upgrade', '20250101000000_0_create_tbl_metrics', 'dev', 0),
+             ('upgrade', '20250101000000_1_insert_into_metrics', 'dev', 1),
+             ('upgrade', '20250101000000_2_create_tbl_events', 'dev', 0),
+             ('upgrade', '20250101000000_3_create_tbl_logs', 'dev', 0)],
         ],
         [
             'prod',
             [
-                'upgrade to 20250101000000_create_tbl_metrics',
-                'upgrade to 20250101000001_insert_into_metrics',
-                'upgrade to 20250101000002_create_tbl_events',
-                'upgrade to 20250101000003_create_tbl_logs',
+                'upgrade to 20250101000000_0_create_tbl_metrics',
+                'upgrade to 20250101000000_1_insert_into_metrics',
+                'upgrade to 20250101000000_2_create_tbl_events',
+                'upgrade to 20250101000000_3_create_tbl_logs',
                 'done',
             ],
             [[(1,)], [(0,)], [(0,)]],
-            [('upgrade', '20250101000000_create_tbl_metrics', 'prod', 0),
-             ('upgrade', '20250101000001_insert_into_metrics', 'prod', 0),
-             ('upgrade', '20250101000002_create_tbl_events', 'prod', 0),
-             ('upgrade', '20250101000003_create_tbl_logs', 'prod', 0)],
+            [('upgrade', '20250101000000_0_create_tbl_metrics', 'prod', 0),
+             ('upgrade', '20250101000000_1_insert_into_metrics', 'prod', 0),
+             ('upgrade', '20250101000000_2_create_tbl_events', 'prod', 0),
+             ('upgrade', '20250101000000_3_create_tbl_logs', 'prod', 0)],
         ],
     ])
     def test_upgrade_downgrade(
@@ -198,48 +202,48 @@ def test_upgrade_downgrade(
 
         # 1 downgrade
         result = subprocess.getoutput('thedus downgrade')
-        self.check_thedus_output(result, ['rollback 20250101000003_create_tbl_logs', 'done'])
+        self.check_thedus_output(result, ['rollback 20250101000000_3_create_tbl_logs', 'done'])
         self.assertEqual(
             [],
             self.clickhouse.execute(
                 f"SELECT * FROM system.tables WHERE table = 'logs' AND database = '{self.db_name}'"
             ))
 
     def test_upgrade_to_revision(self):
-        result = subprocess.getoutput('thedus upgrade 20250101000000_create_tbl_metrics')
-        self.check_thedus_output(result, ['upgrade to 20250101000000_create_tbl_metrics', 'done'])
+        result = subprocess.getoutput('thedus upgrade 20250101000000_0_create_tbl_metrics')
+        self.check_thedus_output(result, ['upgrade to 20250101000000_0_create_tbl_metrics', 'done'])
         self.check_thedus_migration_log([
-            ('upgrade 20250101000000_create_tbl_metrics', '20250101000000_create_tbl_metrics', 'dev', 0),
+            ('upgrade 20250101000000_0_create_tbl_metrics', '20250101000000_0_create_tbl_metrics', 'dev', 0),
         ])
 
         self.assertEqual([(0,)], self.clickhouse.execute('SELECT count() FROM metrics'))
-        result = subprocess.getoutput('thedus upgrade 20250101000002_create_tbl_events')
+        result = subprocess.getoutput('thedus upgrade 20250101000000_2_create_tbl_events')
         self.check_thedus_output(
             result,
             [
-                'SKIP 20250101000001_insert_into_metrics',
-                'upgrade to 20250101000002_create_tbl_events',
+                'SKIP 20250101000000_1_insert_into_metrics',
+                'upgrade to 20250101000000_2_create_tbl_events',
                 'done',
             ])
 
         self.check_thedus_migration_log([
-            ('upgrade 20250101000000_create_tbl_metrics', '20250101000000_create_tbl_metrics', 'dev', 0),
-            ('upgrade 20250101000002_create_tbl_events', '20250101000001_insert_into_metrics', 'dev', 1),
-            ('upgrade 20250101000002_create_tbl_events', '20250101000002_create_tbl_events', 'dev', 0),
+            ('upgrade 20250101000000_0_create_tbl_metrics', '20250101000000_0_create_tbl_metrics', 'dev', 0),
+            ('upgrade 20250101000000_2_create_tbl_events', '20250101000000_1_insert_into_metrics', 'dev', 1),
+            ('upgrade 20250101000000_2_create_tbl_events', '20250101000000_2_create_tbl_events', 'dev', 0),
         ])
 
         self.assertEqual([(0,)], self.clickhouse.execute('SELECT count() FROM events'))
 
     def test_downgrade_to_revision(self):
         subprocess.getoutput('thedus upgrade')
-        result = subprocess.getoutput('thedus downgrade 20250101000001_insert_into_metrics')
+        result = subprocess.getoutput('thedus downgrade 20250101000000_1_insert_into_metrics')
 
         self.check_thedus_output(
             result,
             [
-                'rollback 20250101000003_create_tbl_logs',
-                'rollback 20250101000002_create_tbl_events',
-                'SKIP 20250101000001_insert_into_metrics',
+                'rollback 20250101000000_3_create_tbl_logs',
+                'rollback 20250101000000_2_create_tbl_events',
+                'SKIP 20250101000000_1_insert_into_metrics',
                 'done',
             ]
         )
@@ -253,15 +257,15 @@ def test_downgrade_to_revision(self):
         )
 
         self.check_thedus_migration_log([
-            ('upgrade', '20250101000000_create_tbl_metrics', 'dev', 0),
-            ('upgrade', '20250101000001_insert_into_metrics', 'dev', 1),
-            ('upgrade', '20250101000002_create_tbl_events', 'dev', 0),
-            ('upgrade', '20250101000003_create_tbl_logs', 'dev', 0),
-            ('downgrade 20250101000001_insert_into_metrics', '20250101000002_create_tbl_events', 'dev', 0),
-            ('downgrade 20250101000001_insert_into_metrics', '20250101000001_insert_into_metrics', 'dev', 0),
-            ('downgrade 20250101000001_insert_into_metrics', '20250101000000_create_tbl_metrics', 'dev', 1)])
+            ('upgrade', '20250101000000_0_create_tbl_metrics', 'dev', 0),
+            ('upgrade', '20250101000000_1_insert_into_metrics', 'dev', 1),
+            ('upgrade', '20250101000000_2_create_tbl_events', 'dev', 0),
+            ('upgrade', '20250101000000_3_create_tbl_logs', 'dev', 0),
+            ('downgrade 20250101000000_1_insert_into_metrics', '20250101000000_2_create_tbl_events', 'dev', 0),
+            ('downgrade 20250101000000_1_insert_into_metrics', '20250101000000_1_insert_into_metrics', 'dev', 0),
+            ('downgrade 20250101000000_1_insert_into_metrics', '20250101000000_0_create_tbl_metrics', 'dev', 1)])
 
         result = subprocess.getoutput('thedus downgrade')
-        self.check_thedus_output(result, ['rollback 20250101000000_create_tbl_metrics', 'done'])
+        self.check_thedus_output(result, ['rollback 20250101000000_0_create_tbl_metrics', 'done'])
         result = subprocess.getoutput('thedus downgrade')
         self.check_thedus_output(result, ['done'])

tests/test_cli.py

@@ -1,19 +1,17 @@
 import codecs
 import os
 import re
-from datetime import datetime, timezone
 from typing import List
 
 from .env_manager import EnvManager as Env
 
 
 class MigrationFileManager:
-    @staticmethod
-    def create_migration(name: str) -> str:
-        name = '_'.join([datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S'), name])
-        migration_path = os.path.join(Env.get_thedus_dir(), name) + '.py'
+    MIGRATION_FILENAME_PATTERN = '^[0-9]{14}[a-z0-9_]+.py$'
 
-        with codecs.open(migration_path, 'w', encoding='utf-8') as file:
+    @staticmethod
+    def create_migration(file_path: str) -> str:
+        with codecs.open(file_path, 'w', encoding='utf-8') as file:
             file.write("""from thedus.base_migration import BaseMigration
 
 
@@ -24,23 +22,26 @@ def up(self):
     def down(self):
         self._clickhouse.exec('SELECT 1')
 """)
-            return migration_path
+            return file_path
 
-    @staticmethod
-    def get_migrations(asc: bool = True) -> List[str]:
+    @classmethod
+    def get_migrations(cls, asc: bool = True) -> List[str]:
         """
         Returns a list of all migrations in the thedus directory
         """
         files = []
         for filename in os.listdir(Env.get_thedus_dir()):
-            if not re.fullmatch('^[0-9]{14}[a-z_]+.py$', filename):
-                continue
-
-            files.append(os.path.join(Env.get_thedus_dir(), filename))
+            if cls.is_valid_migration_file(filename):
+                files.append(os.path.join(Env.get_thedus_dir(), filename))
 
         files = sorted(files)
         if asc:
             return files
 
         files.reverse()
         return files
+
+    @classmethod
+    def is_valid_migration_file(cls, filename: str) -> bool:
+        if re.fullmatch(cls.MIGRATION_FILENAME_PATTERN, filename):
+            return True

thedus/migration_file_manager.py

@@ -1,5 +1,6 @@
-import re
+import os
 import logging
+from datetime import timezone, datetime
 from logging.config import dictConfig
 
 import typer
@@ -93,13 +94,17 @@ def state(
 
 @app.command(help='Generates a new migration file. example: thedus create-migration create_metrics')
 def create_migration(name: str):
-    pattern = '[a-z_]{6,30}'
-    if re.fullmatch(pattern, name):
-        migration_path = Migrations.create_migration(name)
+    now = datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')
+    file_name = f'{now}_{name}.py'
+    migration_path = os.path.join(Env.get_thedus_dir(), file_name)
+
+    if Migrations.is_valid_migration_file(file_name):
+        migration_path = Migrations.create_migration(migration_path)
         logging.info(f'{migration_path} created')
         return
 
-    logging.info(f'Invalid migration name "{name}". The name must match reg exp {pattern}')
+    logging.error(f'Invalid migration name "{name}". The name must match reg exp '
+                  f'{Migrations.MIGRATION_FILENAME_PATTERN}')
     exit(1)