[DOP-22140] Add API schemas for SFTP, FTP, FTPS, WebDAV, Samba file sources (#187)

Author: IlyasDevelopment

docs/changelog/next_release/187.feature.rst

@@ -0,0 +1 @@
+Add API schemas for SFTP, FTP, FTPS, WebDAV, Samba file sources

syncmaster/db/models/connection.py

@@ -24,6 +24,11 @@ class ConnectionType(StrEnum):
     MYSQL = "mysql"
     S3 = "s3"
     HDFS = "hdfs"
+    SFTP = "sftp"
+    FTP = "ftp"
+    FTPS = "ftps"
+    WEBDAV = "webdav"
+    SAMBA = "samba"
 
 
 class Connection(Base, ResourceMixin, TimestampMixin):

syncmaster/schemas/v1/auth/__init__.py

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.schemas.v1.auth.basic import (
+    BasicAuthSchema,
+    CreateBasicAuthSchema,
+    ReadBasicAuthSchema,
+    UpdateBasicAuthSchema,
+)
+from syncmaster.schemas.v1.auth.s3 import (
+    CreateS3AuthSchema,
+    ReadS3AuthSchema,
+    S3AuthSchema,
+    UpdateS3AuthSchema,
+)
+from syncmaster.schemas.v1.auth.samba import (
+    CreateSambaAuthSchema,
+    ReadSambaAuthSchema,
+    SambaAuthSchema,
+    UpdateSambaAuthSchema,
+)
+from syncmaster.schemas.v1.auth.token import AuthTokenSchema, TokenPayloadSchema

syncmaster/schemas/v1/auth.py syncmaster/schemas/v1/auth/basic.pysyncmaster/schemas/v1/auth.py renamed to syncmaster/schemas/v1/auth/basic.py

@@ -5,16 +5,6 @@
 from pydantic import BaseModel, SecretStr
 
 
-class TokenPayloadSchema(BaseModel):
-    user_id: int
-
-
-class AuthTokenSchema(BaseModel):
-    access_token: str
-    token_type: str
-    expires_at: float
-
-
 class BasicAuthSchema(BaseModel):
     type: Literal["basic"]
 
@@ -31,21 +21,3 @@ class ReadBasicAuthSchema(BasicAuthSchema):
 class UpdateBasicAuthSchema(BasicAuthSchema):
     user: str | None = None  # noqa: F722
     password: SecretStr | None = None
-
-
-class S3AuthSchema(BaseModel):
-    type: Literal["s3"]
-
-
-class CreateS3AuthSchema(S3AuthSchema):
-    access_key: str
-    secret_key: SecretStr
-
-
-class ReadS3AuthSchema(S3AuthSchema):
-    access_key: str
-
-
-class UpdateS3AuthSchema(S3AuthSchema):
-    access_key: str | None = None
-    secret_key: SecretStr | None = None

syncmaster/schemas/v1/auth/s3.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+from pydantic import BaseModel, SecretStr
+
+
+class S3AuthSchema(BaseModel):
+    type: Literal["s3"]
+
+
+class CreateS3AuthSchema(S3AuthSchema):
+    access_key: str
+    secret_key: SecretStr
+
+
+class ReadS3AuthSchema(S3AuthSchema):
+    access_key: str
+
+
+class UpdateS3AuthSchema(S3AuthSchema):
+    access_key: str | None = None
+    secret_key: SecretStr | None = None

syncmaster/schemas/v1/auth/samba.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+from pydantic import BaseModel, SecretStr
+
+
+class SambaAuthSchema(BaseModel):
+    type: Literal["samba"]
+
+
+class CreateSambaAuthSchema(SambaAuthSchema):
+    user: str
+    password: SecretStr
+    auth_type: Literal["NTLMv1", "NTLMv2"] = "NTLMv2"
+
+
+class ReadSambaAuthSchema(SambaAuthSchema):
+    user: str
+    auth_type: Literal["NTLMv1", "NTLMv2"]
+
+
+class UpdateSambaAuthSchema(SambaAuthSchema):
+    user: str | None = None
+    password: SecretStr | None = None
+    auth_type: Literal["NTLMv1", "NTLMv2"] | None = None

syncmaster/schemas/v1/auth/token.py

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from pydantic import BaseModel
+
+
+class TokenPayloadSchema(BaseModel):
+    user_id: int
+
+
+class AuthTokenSchema(BaseModel):
+    access_token: str
+    token_type: str
+    expires_at: float

syncmaster/schemas/v1/connection_types.py

@@ -10,3 +10,8 @@
 MYSQL_TYPE = Literal["mysql"]
 S3_TYPE = Literal["s3"]
 HDFS_TYPE = Literal["hdfs"]
+SFTP_TYPE = Literal["sftp"]
+FTP_TYPE = Literal["ftp"]
+FTPS_TYPE = Literal["ftps"]
+WEBDAV_TYPE = Literal["webdav"]
+SAMBA_TYPE = Literal["samba"]

syncmaster/schemas/v1/connections/connection.py

@@ -10,6 +10,16 @@
     ReadClickhouseConnectionSchema,
     UpdateClickhouseConnectionSchema,
 )
+from syncmaster.schemas.v1.connections.ftp import (
+    CreateFTPConnectionSchema,
+    ReadFTPConnectionSchema,
+    UpdateFTPConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.ftps import (
+    CreateFTPSConnectionSchema,
+    ReadFTPSConnectionSchema,
+    UpdateFTPSConnectionSchema,
+)
 from syncmaster.schemas.v1.connections.hdfs import (
     CreateHDFSConnectionSchema,
     ReadHDFSConnectionSchema,
@@ -45,6 +55,21 @@
     ReadS3ConnectionSchema,
     UpdateS3ConnectionSchema,
 )
+from syncmaster.schemas.v1.connections.samba import (
+    CreateSambaConnectionSchema,
+    ReadSambaConnectionSchema,
+    UpdateSambaConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.sftp import (
+    CreateSFTPConnectionSchema,
+    ReadSFTPConnectionSchema,
+    UpdateSFTPConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.webdav import (
+    CreateWebDAVConnectionSchema,
+    ReadWebDAVConnectionSchema,
+    UpdateWebDAVConnectionSchema,
+)
 from syncmaster.schemas.v1.page import PageSchema
 from syncmaster.schemas.v1.types import NameConstr
 
@@ -56,7 +81,12 @@
     | CreateClickhouseConnectionSchema
     | CreateHiveConnectionSchema
     | CreateHDFSConnectionSchema
-    | CreateS3ConnectionSchema,
+    | CreateS3ConnectionSchema
+    | CreateSFTPConnectionSchema
+    | CreateFTPConnectionSchema
+    | CreateFTPSConnectionSchema
+    | CreateWebDAVConnectionSchema
+    | CreateSambaConnectionSchema,
     Field(discriminator="type"),
 ]
 ReadConnectionSchema = Annotated[
@@ -67,7 +97,12 @@
     | ReadClickhouseConnectionSchema
     | ReadHiveConnectionSchema
     | ReadHDFSConnectionSchema
-    | ReadS3ConnectionSchema,
+    | ReadS3ConnectionSchema
+    | ReadSFTPConnectionSchema
+    | ReadFTPConnectionSchema
+    | ReadFTPSConnectionSchema
+    | ReadWebDAVConnectionSchema
+    | ReadSambaConnectionSchema,
     Field(discriminator="type"),
 ]
 UpdateConnectionSchema = Annotated[
@@ -78,7 +113,12 @@
     | UpdateClickhouseConnectionSchema
     | UpdateHiveConnectionSchema
     | UpdateHDFSConnectionSchema
-    | UpdateS3ConnectionSchema,
+    | UpdateS3ConnectionSchema
+    | UpdateSFTPConnectionSchema
+    | UpdateFTPConnectionSchema
+    | UpdateFTPSConnectionSchema
+    | UpdateWebDAVConnectionSchema
+    | UpdateSambaConnectionSchema,
     Field(discriminator="type"),
 ]
 

syncmaster/schemas/v1/connections/ftp.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, Field
+
+from syncmaster.schemas.v1.auth import (
+    CreateBasicAuthSchema,
+    ReadBasicAuthSchema,
+    UpdateBasicAuthSchema,
+)
+from syncmaster.schemas.v1.connection_types import FTP_TYPE
+from syncmaster.schemas.v1.connections.connection_base import (
+    CreateConnectionBaseSchema,
+    ReadConnectionBaseSchema,
+    UpdateConnectionBaseSchema,
+)
+
+
+class CreateFTPConnectionDataSchema(BaseModel):
+    host: str
+    port: int
+
+
+class ReadFTPConnectionDataSchema(BaseModel):
+    host: str
+    port: int
+
+
+class UpdateFTPConnectionDataSchema(BaseModel):
+    host: str | None = None
+    port: int | None = None
+
+
+class CreateFTPConnectionSchema(CreateConnectionBaseSchema):
+    type: FTP_TYPE = Field(..., description="Connection type")
+    data: CreateFTPConnectionDataSchema = Field(
+        ...,
+        alias="connection_data",
+        description=(
+            "Data required to connect to the remote server. These are the parameters that are specified in the URL request."
+        ),
+    )
+    auth_data: CreateBasicAuthSchema = Field(
+        description="Credentials for authorization",
+    )
+
+
+class ReadFTPConnectionSchema(ReadConnectionBaseSchema):
+    type: FTP_TYPE
+    data: ReadFTPConnectionDataSchema = Field(alias="connection_data")
+    auth_data: ReadBasicAuthSchema | None = None
+
+
+class UpdateFTPConnectionSchema(UpdateConnectionBaseSchema):
+    type: FTP_TYPE
+    data: UpdateFTPConnectionDataSchema | None = Field(alias="connection_data", default=None)
+    auth_data: UpdateBasicAuthSchema | None = None