[DOP-19907] - change migrations & move models, mixins (#115)

* [DOP-19907] - change migrations & move models, mixins

* [DOP-19907] - recompile migrations

* [DOP-19907] - overwrite rev_id

Author: maxim-lixakov

syncmaster/db/migrations/env.py

@@ -1,15 +1,17 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 import asyncio
+import os
 from logging.config import fileConfig
 
 from alembic import context
+from alembic.script import ScriptDirectory
 from celery.backends.database.session import ResultModelBase
 from sqlalchemy import pool
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
-from syncmaster.db.base import Base
+from syncmaster.db.models import Base
 from syncmaster.settings import Settings
 
 config = context.config
@@ -29,6 +31,39 @@
 )
 
 
+def get_next_revision_id():
+    script_directory = ScriptDirectory.from_config(context.config)
+    versions_path = script_directory.versions
+
+    existing_filenames = os.listdir(versions_path)
+    existing_ids = []
+
+    for filename in existing_filenames:
+        # Assuming filename format: YYYY-MM-DD_XXXX_slug.py
+        parts = filename.split("_")
+        if len(parts) >= 2:
+            id_part = parts[1]
+            try:
+                id_num = int(id_part)
+                existing_ids.append(id_num)
+            except ValueError:
+                continue
+
+    if existing_ids:
+        next_id = max(existing_ids) + 1
+    else:
+        next_id = 1
+
+    return next_id
+
+
+def process_revision_directives(context, revision, directives):
+    if directives:
+        script = directives[0]
+        next_id = get_next_revision_id()
+        script.rev_id = f"{next_id:04d}"
+
+
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
@@ -46,6 +81,7 @@ def run_migrations_offline() -> None:
         url=url,
         target_metadata=target_metadata,
         literal_binds=True,
+        process_revision_directives=process_revision_directives,
         dialect_opts={"paramstyle": "named"},
     )
 
@@ -54,7 +90,11 @@ def run_migrations_offline() -> None:
 
 
 def do_run_migrations(connection: Connection) -> None:
-    context.configure(connection=connection, target_metadata=target_metadata)
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        process_revision_directives=process_revision_directives,
+    )
 
     with context.begin_transaction():
         context.run_migrations()

syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Create user table
+
+Revision ID: 0001
+Revises:
+Create Date: 2023-11-23 11:35:18.193060
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0001"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "user",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("username", sa.String(length=256), nullable=False),
+        sa.Column("is_superuser", sa.Boolean(), nullable=False),
+        sa.Column("is_active", sa.Boolean(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("is_deleted", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk__user")),
+    )
+    op.create_index(op.f("ix__user__username"), "user", ["username"], unique=True)
+
+
+def downgrade():
+    op.drop_index(op.f("ix__user__username"), table_name="user")
+    op.drop_table("user")

syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Create group table
+
+Revision ID: 0002
+Revises: 0001
+Create Date: 2023-11-23 11:36:00.000000
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0002"
+down_revision = "0001"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "group",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("name", sa.String(length=256), nullable=False),
+        sa.Column("description", sa.String(length=512), nullable=False),
+        sa.Column("owner_id", sa.BigInteger(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("is_deleted", sa.Boolean(), nullable=False),
+        sa.Column(
+            "search_vector",
+            postgresql.TSVECTOR(),
+            sa.Computed("to_tsvector('english'::regconfig, name)", persisted=True),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(["owner_id"], ["user.id"], name=op.f("fk__group__owner_id__user"), ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk__group")),
+        sa.UniqueConstraint("name", name=op.f("uq__group__name")),
+    )
+    op.create_index(op.f("ix__group__owner_id"), "group", ["owner_id"], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f("ix__group__owner_id"), table_name="group")
+    op.drop_table("group")

syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Create queue table
+
+Revision ID: 0003
+Revises: 0002
+Create Date: 2023-11-23 11:37:00.000000
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0003"
+down_revision = "0002"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "queue",
+        sa.Column("name", sa.String(length=128), nullable=False),
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("group_id", sa.BigInteger(), nullable=False),
+        sa.Column("description", sa.String(length=512), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("is_deleted", sa.Boolean(), nullable=False),
+        sa.Column(
+            "search_vector",
+            postgresql.TSVECTOR(),
+            sa.Computed("to_tsvector('english'::regconfig, name)", persisted=True),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["group_id"],
+            ["group.id"],
+            name=op.f("fk__queue__group_id__group"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk__queue")),
+        sa.UniqueConstraint("name", name=op.f("uq__queue__name")),
+    )
+    op.create_index(op.f("ix__queue__group_id"), "queue", ["group_id"], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f("ix__queue__group_id"), table_name="queue")
+    op.drop_table("queue")

syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Create connection table
+
+Revision ID: 0004
+Revises: 0003
+Create Date: 2023-11-23 11:38:00.000000
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0004"
+down_revision = "0003"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    sql_expression = (
+        "to_tsvector('english'::regconfig, "
+        "name || ' ' || "
+        "COALESCE(json_extract_path_text(data, 'host'), '') || ' ' || "
+        "COALESCE(translate(json_extract_path_text(data, 'host'), '.', ' '), '')"
+        ")"
+    )
+    op.create_table(
+        "connection",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("group_id", sa.BigInteger(), nullable=False),
+        sa.Column("name", sa.String(length=128), nullable=False),
+        sa.Column("description", sa.String(length=512), nullable=False),
+        sa.Column("data", sa.JSON(), nullable=False),
+        sa.Column("is_deleted", sa.Boolean(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("search_vector", postgresql.TSVECTOR(), sa.Computed(sql_expression, persisted=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["group_id"],
+            ["group.id"],
+            name=op.f("fk__connection__group_id__group"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk__connection")),
+        sa.UniqueConstraint("name", "group_id", name=op.f("uq__connection__name_group_id")),
+    )
+    op.create_index(op.f("ix__connection__group_id"), "connection", ["group_id"], unique=False)
+    op.create_index(
+        "idx_connection_search_vector",
+        "connection",
+        ["search_vector"],
+        unique=False,
+        postgresql_using="gin",
+    )
+
+
+def downgrade():
+    op.drop_index(op.f("ix__connection__group_id"), table_name="connection")
+    op.drop_index("idx_connection_search_vector", table_name="connection", postgresql_using="gin")
+    op.drop_table("connection")

syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Create user_group table
+
+Revision ID: 0005
+Revises: 0004
+Create Date: 2023-11-23 11:39:00.000000
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0005"
+down_revision = "0004"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "user_group",
+        sa.Column("user_id", sa.BigInteger(), nullable=False),
+        sa.Column("group_id", sa.BigInteger(), nullable=False),
+        sa.Column("role", sa.String(255), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["group_id"],
+            ["group.id"],
+            name=op.f("fk__user_group__group_id__group"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+            name=op.f("fk__user_group__user_id__user"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("user_id", "group_id", name=op.f("pk__user_group")),
+    )
+    op.create_index(op.f("ix__user_group__group_id"), "user_group", ["group_id"], unique=False)
+    op.create_index(op.f("ix__user_group__user_id"), "user_group", ["user_id"], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f("ix__user_group__user_id"), table_name="user_group")
+    op.drop_index(op.f("ix__user_group__group_id"), table_name="user_group")
+    op.drop_table("user_group")

syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Create auth_data table
+
+Revision ID: 0006
+Revises: 0005
+Create Date: 2023-11-23 11:40:00.000000
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0006"
+down_revision = "0005"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "auth_data",
+        sa.Column("connection_id", sa.BigInteger(), nullable=False),
+        sa.Column("value", sa.String(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["connection_id"],
+            ["connection.id"],
+            name=op.f("fk__auth_data__connection_id__connection"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("connection_id", name=op.f("pk__auth_data")),
+    )
+
+
+def downgrade():
+    op.drop_table("auth_data")