[DOP-19902] Add MySQL API schema

Ilyas Gasanov · Ilyas Gasanov · commit e843320f7ea1 · 2024-11-12T20:20:18.000+03:00
diff --git a/README.rst b/README.rst
@@ -37,6 +37,7 @@ List of currently supported connections:
 * Postgres
 * Oracle
 * MSSQL
+* MySQL
 * HDFS
 * S3
 
diff --git a/docs/changelog/next_release/126.feature.rst b/docs/changelog/next_release/126.feature.rst
@@ -0,0 +1 @@
+Add MySQL API schema
diff --git a/syncmaster/backend/api/v1/connections.py b/syncmaster/backend/api/v1/connections.py
@@ -22,6 +22,7 @@
     HDFS_TYPE,
     HIVE_TYPE,
     MSSQL_TYPE,
+    MYSQL_TYPE,
     ORACLE_TYPE,
     POSTGRES_TYPE,
     S3_TYPE,
@@ -39,7 +40,7 @@
 
 router = APIRouter(tags=["Connections"], responses=get_error_responses())
 
-CONNECTION_TYPES = ORACLE_TYPE, POSTGRES_TYPE, CLICKHOUSE_TYPE, HIVE_TYPE, MSSQL_TYPE, S3_TYPE, HDFS_TYPE
+CONNECTION_TYPES = ORACLE_TYPE, POSTGRES_TYPE, CLICKHOUSE_TYPE, HIVE_TYPE, MSSQL_TYPE, MYSQL_TYPE, S3_TYPE, HDFS_TYPE
 
 
 @router.get("/connections")
diff --git a/syncmaster/schemas/v1/connection_types.py b/syncmaster/schemas/v1/connection_types.py
@@ -8,6 +8,7 @@
 POSTGRES_TYPE = Literal["postgres"]
 CLICKHOUSE_TYPE = Literal["clickhouse"]
 MSSQL_TYPE = Literal["mssql"]
+MYSQL_TYPE = Literal["mysql"]
 S3_TYPE = Literal["s3"]
 HDFS_TYPE = Literal["hdfs"]
 
@@ -18,5 +19,6 @@ class ConnectionType(str, Enum):
     ORACLE = "oracle"
     CLICKHOUSE = "clickhouse"
     MSSQL = "mssql"
+    MYSQL = "mysql"
     S3 = "s3"
     HDFS = "hdfs"
diff --git a/syncmaster/schemas/v1/connections/connection.py b/syncmaster/schemas/v1/connections/connection.py
@@ -35,6 +35,14 @@
     UpdateMSSQLAuthSchema,
     UpdateMSSQLConnectionSchema,
 )
+from syncmaster.schemas.v1.connections.mysql import (
+    CreateMySQLAuthSchema,
+    CreateMySQLConnectionSchema,
+    ReadMySQLAuthSchema,
+    ReadMySQLConnectionSchema,
+    UpdateMySQLAuthSchema,
+    UpdateMySQLConnectionSchema,
+)
 from syncmaster.schemas.v1.connections.oracle import (
     CreateOracleAuthSchema,
     CreateOracleConnectionSchema,
@@ -69,6 +77,7 @@
     | ReadPostgresConnectionSchema
     | ReadClickhouseConnectionSchema
     | ReadMSSQLConnectionSchema
+    | ReadMySQLConnectionSchema
     | S3ReadConnectionSchema
 )
 CreateConnectionDataSchema = (
@@ -77,6 +86,7 @@
     | CreatePostgresConnectionSchema
     | CreateClickhouseConnectionSchema
     | CreateMSSQLConnectionSchema
+    | CreateMySQLConnectionSchema
     | HDFSCreateConnectionSchema
     | S3CreateConnectionSchema
 )
@@ -88,13 +98,15 @@
     | UpdatePostgresConnectionSchema
     | UpdateClickhouseConnectionSchema
     | UpdateMSSQLConnectionSchema
+    | UpdateMySQLConnectionSchema
 )
 ReadConnectionAuthDataSchema = (
     ReadHiveAuthSchema
     | ReadOracleAuthSchema
     | ReadPostgresAuthSchema
     | ReadClickhouseAuthSchema
     | ReadMSSQLAuthSchema
+    | ReadMySQLAuthSchema
     | S3ReadAuthSchema
     | HDFSReadAuthSchema
 )
@@ -104,6 +116,7 @@
     | CreatePostgresAuthSchema
     | CreateClickhouseAuthSchema
     | CreateMSSQLAuthSchema
+    | CreateMySQLAuthSchema
     | S3CreateAuthSchema
     | HDFSCreateAuthSchema
 )
@@ -113,6 +126,7 @@
     | UpdatePostgresAuthSchema
     | UpdateClickhouseAuthSchema
     | UpdateMSSQLAuthSchema
+    | UpdateMySQLAuthSchema
     | S3UpdateAuthSchema
     | HDFSUpdateAuthSchema
 )
diff --git a/syncmaster/schemas/v1/connections/mysql.py b/syncmaster/schemas/v1/connections/mysql.py
@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from pydantic import BaseModel, Field, SecretStr
+
+from syncmaster.schemas.v1.connection_types import MYSQL_TYPE
+
+
+class MySQLBaseSchema(BaseModel):
+    type: MYSQL_TYPE
+
+    class Config:
+        from_attributes = True
+
+
+class ReadMySQLConnectionSchema(MySQLBaseSchema):
+    host: str
+    port: int
+    database_name: str
+    additional_params: dict = Field(default_factory=dict)
+
+
+class ReadMySQLAuthSchema(MySQLBaseSchema):
+    user: str
+
+
+class UpdateMySQLConnectionSchema(MySQLBaseSchema):
+    host: str | None = None
+    port: int | None = None
+    database_name: str | None = None
+    additional_params: dict | None = Field(default_factory=dict)
+
+
+class UpdateMySQLAuthSchema(MySQLBaseSchema):
+    user: str | None = None  # noqa: F722
+    password: SecretStr | None = None
+
+
+class CreateMySQLConnectionSchema(MySQLBaseSchema):
+    host: str
+    port: int
+    database_name: str
+    additional_params: dict = Field(default_factory=dict)
+
+
+class CreateMySQLAuthSchema(MySQLBaseSchema):
+    user: str
+    password: SecretStr
diff --git a/syncmaster/schemas/v1/transfers/__init__.py b/syncmaster/schemas/v1/transfers/__init__.py
@@ -10,6 +10,7 @@
     ClickhouseReadTransferSourceAndTarget,
     HiveReadTransferSourceAndTarget,
     MSSQLReadTransferSourceAndTarget,
+    MySQLReadTransferSourceAndTarget,
     OracleReadTransferSourceAndTarget,
     PostgresReadTransferSourceAndTarget,
 )
@@ -35,6 +36,7 @@
     | OracleReadTransferSourceAndTarget
     | ClickhouseReadTransferSourceAndTarget
     | MSSQLReadTransferSourceAndTarget
+    | MySQLReadTransferSourceAndTarget
     | S3ReadTransferSource
 )
 
@@ -45,6 +47,7 @@
     | OracleReadTransferSourceAndTarget
     | ClickhouseReadTransferSourceAndTarget
     | MSSQLReadTransferSourceAndTarget
+    | MySQLReadTransferSourceAndTarget
     | S3ReadTransferTarget
 )
 
@@ -55,6 +58,7 @@
     | OracleReadTransferSourceAndTarget
     | ClickhouseReadTransferSourceAndTarget
     | MSSQLReadTransferSourceAndTarget
+    | MySQLReadTransferSourceAndTarget
     | S3CreateTransferSource
 )
 
@@ -65,6 +69,7 @@
     | OracleReadTransferSourceAndTarget
     | ClickhouseReadTransferSourceAndTarget
     | MSSQLReadTransferSourceAndTarget
+    | MySQLReadTransferSourceAndTarget
     | S3CreateTransferTarget
 )
 
@@ -75,6 +80,7 @@
     | OracleReadTransferSourceAndTarget
     | ClickhouseReadTransferSourceAndTarget
     | MSSQLReadTransferSourceAndTarget
+    | MySQLReadTransferSourceAndTarget
     | S3CreateTransferSource
     | None
 )
@@ -86,6 +92,7 @@
     | OracleReadTransferSourceAndTarget
     | ClickhouseReadTransferSourceAndTarget
     | MSSQLReadTransferSourceAndTarget
+    | MySQLReadTransferSourceAndTarget
     | S3CreateTransferTarget
     | None
 )
diff --git a/syncmaster/schemas/v1/transfers/db.py b/syncmaster/schemas/v1/transfers/db.py
@@ -8,6 +8,7 @@
     CLICKHOUSE_TYPE,
     HIVE_TYPE,
     MSSQL_TYPE,
+    MYSQL_TYPE,
     ORACLE_TYPE,
     POSTGRES_TYPE,
 )
@@ -35,3 +36,7 @@ class ClickhouseReadTransferSourceAndTarget(ReadDBTransfer):
 
 class MSSQLReadTransferSourceAndTarget(ReadDBTransfer):
     type: MSSQL_TYPE
+
+
+class MySQLReadTransferSourceAndTarget(ReadDBTransfer):
+    type: MYSQL_TYPE
diff --git a/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py b/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py
@@ -39,7 +39,7 @@ async def group_connections(
                     "bucket": "bucket",
                 },
             )
-        elif conn_type == ConnectionType.POSTGRES:
+        elif conn_type in [ConnectionType.POSTGRES, ConnectionType.MYSQL]:
             new_data.update(
                 {
                     "database_name": "database",
diff --git a/tests/test_unit/test_connections/test_create_connection.py b/tests/test_unit/test_connections/test_create_connection.py
@@ -282,7 +282,7 @@ async def test_check_fields_validation_on_create_connection(
                     "context": {
                         "discriminator": "'type'",
                         "tag": "POSTGRESQL",
-                        "expected_tags": "'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'hdfs', 's3'",
+                        "expected_tags": "'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'mysql', 'hdfs', 's3'",
                     },
                     "input": {
                         "type": "POSTGRESQL",
@@ -292,7 +292,7 @@ async def test_check_fields_validation_on_create_connection(
                         "database_name": "postgres",
                     },
                     "location": ["body", "connection_data"],
-                    "message": "Input tag 'POSTGRESQL' found using 'type' does not match any of the expected tags: 'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'hdfs', 's3'",
+                    "message": "Input tag 'POSTGRESQL' found using 'type' does not match any of the expected tags: 'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'mysql', 'hdfs', 's3'",
                     "code": "union_tag_invalid",
                 },
             ],
diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py
@@ -0,0 +1,80 @@
+import pytest
+from httpx import AsyncClient
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from syncmaster.db.models import AuthData, Connection
+from syncmaster.db.repositories.utils import decrypt_auth_data
+from syncmaster.settings import Settings
+from tests.mocks import MockGroup, UserTestRoles
+
+pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.mysql]
+
+
+async def test_developer_plus_can_create_mysql_connection(
+    client: AsyncClient,
+    group: MockGroup,
+    session: AsyncSession,
+    settings: Settings,
+    role_developer_plus: UserTestRoles,
+):
+    # Arrange
+    user = group.get_member_of_role(role_developer_plus)
+
+    # Act
+    result = await client.post(
+        "v1/connections",
+        headers={"Authorization": f"Bearer {user.token}"},
+        json={
+            "group_id": group.id,
+            "name": "New connection",
+            "description": "",
+            "connection_data": {
+                "type": "mysql",
+                "host": "127.0.0.1",
+                "port": 3306,
+                "database_name": "database",
+            },
+            "auth_data": {
+                "type": "mysql",
+                "user": "user",
+                "password": "secret",
+            },
+        },
+    )
+    connection = (
+        await session.scalars(
+            select(Connection).filter_by(
+                name="New connection",
+            ),
+        )
+    ).first()
+
+    creds = (
+        await session.scalars(
+            select(AuthData).filter_by(
+                connection_id=connection.id,
+            ),
+        )
+    ).one()
+
+    # Assert
+    decrypted = decrypt_auth_data(creds.value, settings=settings)
+    assert result.status_code == 200
+    assert result.json() == {
+        "id": connection.id,
+        "name": connection.name,
+        "description": connection.description,
+        "group_id": connection.group_id,
+        "connection_data": {
+            "type": connection.data["type"],
+            "host": connection.data["host"],
+            "port": connection.data["port"],
+            "database_name": connection.data["database_name"],
+            "additional_params": connection.data["additional_params"],
+        },
+        "auth_data": {
+            "type": decrypted["type"],
+            "user": decrypted["user"],
+        },
+    }
diff --git a/tests/test_unit/test_connections/test_db_connection/test_update_mysql_connection.py b/tests/test_unit/test_connections/test_db_connection/test_update_mysql_connection.py
diff --git a/tests/test_unit/test_connections/test_read_connections.py b/tests/test_unit/test_connections/test_read_connections.py
diff --git a/tests/test_unit/test_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_create_transfer.py
diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ async def group_connections(`
`39`	`39`	`"bucket": "bucket",`
`40`	`40`	`},`
`41`	`41`	`)`
`42`		`- elif conn_type == ConnectionType.POSTGRES:`
	`42`	`+ elif conn_type in [ConnectionType.POSTGRES, ConnectionType.MYSQL]:`
`43`	`43`	`new_data.update(`
`44`	`44`	`{`
`45`	`45`	`"database_name": "database",`