Add many-to-many to plaforms-sign_keys (AlmaLinux/build-system#370) (#1041)

* Add many-to-many to plaforms-sign_keys (AlmaLinux/build-system#370)

- alembic migration
- change in models
- added a test for one platform with 2 keys

* Changing POST sign_keys schema to accept list of platforms + bug fixes

* Important bug fix for alembic downgrade:

- Disable delete on cascade for downgrade to preserve referencing tables

* Adding platform_ids to /sign-keys/ response

Author: bklvsky

alws/alembic/versions/63bc4e0b699f_add_many_to_many_relationship_between_.py

@@ -0,0 +1,142 @@
+"""Add many to many relationship between platforms and sign_keys
+
+Revision ID: 63bc4e0b699f
+Revises: 764a67b23038
+Create Date: 2024-11-01 12:26:26.326314
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.engine import reflection
+
+# revision identifiers, used by Alembic.
+revision = '63bc4e0b699f'
+down_revision = '9977cc722e24'
+branch_labels = None
+depends_on = None
+
+
+def disable_fk_checks():
+    op.execute(sa.text("SET session_replication_role = 'replica'"))
+
+def enable_fk_checks():
+    op.execute(sa.text("SET session_replication_role = 'origin'"))
+
+def get_columns(table_name):
+    # Use SQLAlchemy's Inspector to retrieve the columns of the specified table
+    conn = op.get_bind()
+    inspector = reflection.Inspector.from_engine(conn)
+    columns = [col["name"] for col in inspector.get_columns(table_name)]
+    return columns
+
+
+def create_backup_table(table_name):
+    backup_table = f'{table_name}_backup'
+    op.execute(
+        sa.text(f"CREATE TABLE {backup_table} AS TABLE {table_name} WITH DATA")
+    )
+
+
+def drop_backup_table(backup_table_name):
+    op.execute(sa.text(f"DROP TABLE IF EXISTS {backup_table_name}"))
+
+
+def restore_from_backup(table_name):
+    backup_table = f'{table_name}_backup'
+    columns = get_columns(table_name)
+    column_list = ', '.join(columns)
+    disable_fk_checks()
+    op.execute(sa.text(f"DELETE FROM {table_name}"))
+    op.execute(
+        sa.text(
+            f"INSERT INTO {table_name} ({column_list}) SELECT {column_list} FROM {backup_table}"
+        )
+    )
+    enable_fk_checks()
+
+
+def create_association_table():
+    op.create_table(
+        'platforms_sign_keys',
+        sa.Column(
+            'platform_id',
+            sa.Integer,
+            sa.ForeignKey('platforms.id', ondelete="CASCADE"),
+            primary_key=True,
+        ),
+        sa.Column(
+            'sign_key_id',
+            sa.Integer,
+            sa.ForeignKey('sign_keys.id', ondelete="CASCADE"),
+            primary_key=True,
+        ),
+    )
+
+
+def upgrade():
+    create_backup_table("platforms")
+    create_backup_table("sign_keys")
+    op.drop_constraint(
+        'sign_keys_platform_id_fkey', 'sign_keys', type_='foreignkey'
+    )
+    create_association_table()
+    op.execute(
+        sa.text(
+            """
+        INSERT INTO platforms_sign_keys (platform_id, sign_key_id)
+        SELECT platform_id, id FROM sign_keys
+        WHERE platform_id IS NOT NULL
+    """
+        )
+    )
+    op.drop_column("sign_keys", "platform_id")
+
+
+def downgrade():
+    op.add_column(
+        "sign_keys", sa.Column("platform_id", sa.Integer, nullable=True)
+    )
+    op.execute(
+        sa.text(
+            """
+        UPDATE sign_keys
+        SET platform_id = (
+            SELECT platform_id
+            FROM platforms_sign_keys
+            WHERE platforms_sign_keys.sign_key_id = sign_keys.id
+            LIMIT 1
+        )
+    """
+        )
+    )
+    op.create_foreign_key(
+        'sign_keys_platform_id_fkey',
+        'sign_keys',
+        'platforms',
+        ['platform_id'],
+        ['id'],
+    )
+    op.drop_table('platforms_sign_keys')
+
+    restore_from_backup("platforms")
+    restore_from_backup("sign_keys")
+
+    # Neccessary to preserve id sequences
+    op.execute(
+        sa.text(
+            """
+            SELECT setval('platforms_id_seq', MAX(id)) FROM platforms;
+            """
+        )
+    )
+    op.execute(
+        sa.text(
+            """
+            SELECT setval('sign_keys_id_seq', MAX(id)) FROM sign_keys;
+            """
+        )
+    )
+
+    drop_backup_table("platforms_backup")
+    drop_backup_table("sign_keys_backup")

alws/crud/sign_key.py

@@ -1,10 +1,11 @@
+import datetime
+import logging
 import typing
 
+from sqlalchemy import and_, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
-from sqlalchemy import and_, update
 from sqlalchemy.orm import selectinload
-import datetime
 
 from alws import models
 from alws.crud.user import get_user
@@ -25,11 +26,14 @@ async def get_sign_keys(
 ) -> typing.List[models.SignKey]:
     limited_user = await get_user(db, user.id)
     result = await db.execute(
-        select(models.SignKey).where(models.SignKey.active).options(
+        select(models.SignKey)
+        .where(models.SignKey.active)
+        .options(
             selectinload(models.SignKey.owner),
             selectinload(models.SignKey.roles).selectinload(
                 models.UserRole.actions
             ),
+            selectinload(models.SignKey.platforms),
         )
     )
     suitable_keys = [
@@ -40,6 +44,17 @@ async def get_sign_keys(
     return suitable_keys
 
 
+async def get_sign_key(db: AsyncSession, key_id: int):
+    sign_key = await db.execute(
+        select(models.SignKey)
+        .where(models.SignKey.keyid == key_id)
+        .options(selectinload(models.SignKey.platforms))
+    )
+    if not sign_key:
+        raise DataNotFoundError(f"Sign key with ID {key_id} does not exist")
+    return sign_key.scalars().first()
+
+
 async def create_sign_key(
     db: AsyncSession, payload: sign_schema.SignKeyCreate
 ) -> models.SignKey:
@@ -50,21 +65,34 @@ async def create_sign_key(
         raise SignKeyAlreadyExistsError(
             f"Key with keyid {payload.keyid} already exists"
         )
-    if payload.platform_id:
-        check_platform = await db.execute(
-            select(models.Platform.id).where(
-                models.Platform.id == payload.platform_id
-            )
+    model = payload.model_dump()
+    platform_ids = model.pop('platform_ids', None)
+    sign_key = models.SignKey(**model)
+
+    if platform_ids:
+        check_platforms = await db.execute(
+            select(models.Platform).where(models.Platform.id.in_(platform_ids))
         )
-        if not check_platform.scalars().first():
+        platform_instances = check_platforms.scalars().all()
+        if not platform_instances:
             raise PlatformMissingError(
-                f"No platform with id '{payload.platform_id}' "
-                "exists in the system"
+                f"No platforms with ids '{platform_ids}' exist in the system"
+            )
+        if len(platform_instances) < len(platform_ids):
+            db_platform_ids = [pl.id for pl in platform_instances]
+            missing_platform_ids = [
+                platform_id
+                for platform_id in platform_ids
+                if platform_id not in db_platform_ids
+            ]
+            logging.warning(
+                f"Platforms with ids: '{missing_platform_ids}' "
+                "are missing in the system. Did not add them to the sign key."
             )
-    sign_key = models.SignKey(**payload.model_dump())
+        sign_key.platforms = platform_instances
     db.add(sign_key)
     await db.flush()
-    await db.refresh(sign_key)
+    await db.refresh(sign_key, attribute_names=['platforms'])
     return sign_key
 
 

alws/models.py

@@ -230,6 +230,23 @@ def finished_at(cls) -> Mapped[Optional[datetime.datetime]]:
     ),
 )
 
+platforms_sign_keys = sqlalchemy.Table(
+    "platforms_sign_keys",
+    Base.metadata,
+    sqlalchemy.Column(
+        "platform_id",
+        sqlalchemy.Integer,
+        sqlalchemy.ForeignKey("platforms.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+    sqlalchemy.Column(
+        "sign_key_id",
+        sqlalchemy.Integer,
+        sqlalchemy.ForeignKey("sign_keys.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+)
+
 
 class Platform(PermissionsMixin, Base):
     __tablename__ = "platforms"
@@ -244,7 +261,9 @@ class Platform(PermissionsMixin, Base):
     type: Mapped[str] = mapped_column(sqlalchemy.Text, nullable=False)
     distr_type: Mapped[str] = mapped_column(sqlalchemy.Text, nullable=False)
     distr_version: Mapped[str] = mapped_column(sqlalchemy.Text, nullable=False)
-    pgp_key: Mapped[Optional[str]] = mapped_column(sqlalchemy.Text, nullable=True)
+    pgp_key: Mapped[Optional[str]] = mapped_column(
+        sqlalchemy.Text, nullable=True
+    )
     module_build_index: Mapped[int] = mapped_column(
         sqlalchemy.Integer, default=1
     )
@@ -281,7 +300,7 @@ class Platform(PermissionsMixin, Base):
         "Repository", secondary=PlatformRepo
     )
     sign_keys: Mapped[List["SignKey"]] = relationship(
-        "SignKey", back_populates="platform"
+        "SignKey", secondary=platforms_sign_keys, back_populates="platforms"
     )
     roles: Mapped[List["UserRole"]] = relationship(
         "UserRole", secondary=PlatformRoleMapping
@@ -953,7 +972,7 @@ class Team(PermissionsMixin, Base):
         "Product", back_populates="team"
     )
     test_repositories: Mapped[List["TestRepository"]] = relationship(
-         "TestRepository", back_populates="team"
+        "TestRepository", back_populates="team"
     )
     roles: Mapped[List["UserRole"]] = relationship(
         "UserRole",
@@ -1199,7 +1218,9 @@ class TestRepository(PermissionsMixin, TeamMixin, Base):
         back_populates="test_repository",
         cascade="all, delete",
     )
-    team: Mapped["Team"] = relationship("Team", back_populates="test_repositories")
+    team: Mapped["Team"] = relationship(
+        "Team", back_populates="test_repositories"
+    )
     roles: Mapped[List["UserRole"]] = relationship(
         "UserRole", secondary=TestRepositoryRoleMapping
     )
@@ -1300,11 +1321,10 @@ class SignKey(PermissionsMixin, Base):
     inserted: Mapped[datetime.datetime] = mapped_column(
         sqlalchemy.DateTime, default=datetime.datetime.utcnow()
     )
-    active: Mapped[bool] = mapped_column(
-        sqlalchemy.Boolean, default=True
-    )
+    active: Mapped[bool] = mapped_column(sqlalchemy.Boolean, default=True)
     archived: Mapped[datetime.datetime] = mapped_column(
-        sqlalchemy.DateTime, nullable=True,
+        sqlalchemy.DateTime,
+        nullable=True,
     )
     product_id: Mapped[Optional[int]] = mapped_column(
         sqlalchemy.Integer,
@@ -1317,16 +1337,8 @@ class SignKey(PermissionsMixin, Base):
     product: Mapped["Product"] = relationship(
         'Product', back_populates='sign_keys'
     )
-    platform_id: Mapped[Optional[int]] = mapped_column(
-        sqlalchemy.Integer,
-        sqlalchemy.ForeignKey(
-            "platforms.id",
-            name="sign_keys_platform_id_fkey",
-        ),
-        nullable=True,
-    )
-    platform: Mapped["Platform"] = relationship(
-        "Platform", back_populates="sign_keys"
+    platforms: Mapped[List["Platform"]] = relationship(
+        "Platform", secondary=platforms_sign_keys, back_populates="sign_keys"
     )
     build_task_artifacts: Mapped[List["BuildTaskArtifact"]] = relationship(
         "BuildTaskArtifact",

alws/routers/sign_key.py

@@ -27,7 +27,11 @@ async def get_sign_keys(
     db: AsyncSession = Depends(AsyncSessionDependency(key=get_async_db_key())),
     user=Depends(get_current_user),
 ):
-    return await sign_key.get_sign_keys(db, user)
+    keys = await sign_key.get_sign_keys(db, user)
+    for key in keys:
+        if key.platforms:
+            key.platform_ids = [pl.id for pl in key.platforms]
+    return keys
 
 
 @router.post(
@@ -42,7 +46,10 @@ async def create_sign_key(
 ):
     try:
         payload.owner_id = user.id
-        return await sign_key.create_sign_key(db, payload)
+        key = await sign_key.create_sign_key(db, payload)
+        if payload.platform_ids:
+            key.platform_ids = [pl.id for pl in key.platforms]
+        return key
     except (PlatformMissingError, SignKeyAlreadyExistsError) as e:
         raise HTTPException(status.HTTP_400_BAD_REQUEST, str(e))
 

alws/schemas/sign_schema.py

@@ -13,7 +13,7 @@ class SignKey(BaseModel):
     inserted: datetime
     active: bool = True
     archived: typing.Optional[datetime] = None
-    platform_id: typing.Optional[int] = None
+    platform_ids: typing.Optional[typing.List[int]] = None
     product_id: typing.Optional[int] = None
 
     class Config:
@@ -26,7 +26,7 @@ class SignKeyCreate(BaseModel):
     keyid: str
     fingerprint: str
     public_url: str
-    platform_id: typing.Optional[int] = None
+    platform_ids: typing.Optional[typing.List[int]] = None
     owner_id: typing.Optional[int] = None
 
 

tests/fixtures/sign_keys.py

@@ -26,6 +26,18 @@ def basic_sign_key_payload() -> dict:
     }
 
 
+@pytest.fixture
+def additional_sign_key_payload() -> dict:
+    keyid = "01234567890ABCDE"
+    return {
+        "name": "Test key",
+        "description": "Test sign key",
+        "keyid": keyid,
+        "fingerprint": "01234567890ABCDEF1234567890ABCDEF1234567",
+        "public_url": "no_url",
+    }
+
+
 async def __create_sign_key(
     async_session: AsyncSession, payload: dict
 ) -> SignKey: