Skip to content

Commit 7f0eaae

Browse files
dolfinusdolfinus
committed
[DOP-30631] Small OpenAPI schema fixes
1 parent 7e2c59c commit 7f0eaae

File tree

16 files changed

+226
-188
lines changed

16 files changed

+226
-188
lines changed

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ max-cognitive-score = 20
281281
# Max amount of cognitive complexity per module
282282
max-cognitive-average = 25
283283
max-imports = 25
284-
max-imported-names = 50
284+
max-imported-names = 55
285285
# Max of expression usages in a module
286286
max-module-expressions = 15
287287
# Max of expression usages in a function

‎syncmaster/schemas/v1/__init__.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,6 @@
2929
ReadTransferSchema,
3030
TransferPageSchema,
3131
)
32-
from syncmaster.schemas.v1.transfers.db import (
33-
HiveReadTransferSourceAndTarget,
34-
OracleReadTransferSourceAndTarget,
35-
PostgresReadTransferSourceAndTarget,
36-
ReadDBTransfer,
37-
)
3832
from syncmaster.schemas.v1.transfers.file_format import (
3933
CSV,
4034
JSON,
@@ -87,10 +81,6 @@
8781
"ShortRunSchema",
8882
"FullStrategy",
8983
"IncrementalStrategy",
90-
"ReadDBTransfer",
91-
"HiveReadTransferSourceAndTarget",
92-
"OracleReadTransferSourceAndTarget",
93-
"PostgresReadTransferSourceAndTarget",
9484
"CSV",
9585
"JSON",
9686
"ORC",

‎syncmaster/schemas/v1/connection_types.py‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,37 @@
1818
SAMBA_TYPE = Literal["samba"]
1919

2020
CONNECTION_TYPES = [
21-
"oracle",
22-
"postgres",
2321
"clickhouse",
2422
"hive",
2523
"iceberg_rest_s3",
2624
"mssql",
2725
"mysql",
28-
"s3",
26+
"oracle",
27+
"postgres",
28+
"ftp",
29+
"ftps",
2930
"hdfs",
31+
"s3",
32+
"samba",
3033
"sftp",
34+
"webdav",
35+
]
36+
FILE_CONNECTION_TYPES = [
3137
"ftp",
3238
"ftps",
33-
"webdav",
39+
"hdfs",
40+
"s3",
3441
"samba",
42+
"sftp",
43+
"webdav",
44+
]
45+
DB_CONNECTION_TYPES = [
46+
"clickhouse",
47+
"hive",
48+
"iceberg_rest_s3",
49+
"mssql",
50+
"mysql",
51+
"oracle",
52+
"postgres",
3553
]
36-
FILE_CONNECTION_TYPES = ["s3", "hdfs", "sftp", "ftp", "ftps", "webdav", "samba"]
37-
DB_CONNECTION_TYPES = ["oracle", "postgres", "clickhouse", "hive", "iceberg_rest_s3", "mssql", "mysql"]
54+
CONNECTION_TYPES = [*DB_CONNECTION_TYPES, *FILE_CONNECTION_TYPES]

‎syncmaster/schemas/v1/connections/connection.py‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -79,54 +79,54 @@
7979
from syncmaster.schemas.v1.types import NameConstr
8080

8181
CreateConnectionSchema = Annotated[
82-
CreateOracleConnectionSchema
83-
| CreatePostgresConnectionSchema
84-
| CreateMySQLConnectionSchema
85-
| CreateMSSQLConnectionSchema
86-
| CreateClickhouseConnectionSchema
82+
CreateClickhouseConnectionSchema
8783
| CreateHiveConnectionSchema
8884
| CreateIcebergConnectionSchema
85+
| CreateMSSQLConnectionSchema
86+
| CreateMySQLConnectionSchema
87+
| CreateOracleConnectionSchema
88+
| CreatePostgresConnectionSchema
89+
| CreateFTPConnectionSchema
90+
| CreateFTPSConnectionSchema
8991
| CreateHDFSConnectionSchema
9092
| CreateS3ConnectionSchema
93+
| CreateSambaConnectionSchema
9194
| CreateSFTPConnectionSchema
92-
| CreateFTPConnectionSchema
93-
| CreateFTPSConnectionSchema
94-
| CreateWebDAVConnectionSchema
95-
| CreateSambaConnectionSchema,
95+
| CreateWebDAVConnectionSchema,
9696
Field(discriminator="type"),
9797
]
9898
ReadConnectionSchema = Annotated[
99-
ReadOracleConnectionSchema
100-
| ReadPostgresConnectionSchema
101-
| ReadMySQLConnectionSchema
102-
| ReadMSSQLConnectionSchema
103-
| ReadClickhouseConnectionSchema
99+
ReadClickhouseConnectionSchema
104100
| ReadHiveConnectionSchema
105101
| ReadIcebergConnectionSchema
102+
| ReadMSSQLConnectionSchema
103+
| ReadMySQLConnectionSchema
104+
| ReadOracleConnectionSchema
105+
| ReadPostgresConnectionSchema
106+
| ReadFTPConnectionSchema
107+
| ReadFTPSConnectionSchema
106108
| ReadHDFSConnectionSchema
107109
| ReadS3ConnectionSchema
110+
| ReadSambaConnectionSchema
108111
| ReadSFTPConnectionSchema
109-
| ReadFTPConnectionSchema
110-
| ReadFTPSConnectionSchema
111-
| ReadWebDAVConnectionSchema
112-
| ReadSambaConnectionSchema,
112+
| ReadWebDAVConnectionSchema,
113113
Field(discriminator="type"),
114114
]
115115
UpdateConnectionSchema = Annotated[
116-
UpdateOracleConnectionSchema
117-
| UpdatePostgresConnectionSchema
118-
| UpdateMySQLConnectionSchema
119-
| UpdateMSSQLConnectionSchema
120-
| UpdateClickhouseConnectionSchema
116+
UpdateClickhouseConnectionSchema
121117
| UpdateHiveConnectionSchema
122118
| UpdateIcebergConnectionSchema
119+
| UpdateMSSQLConnectionSchema
120+
| UpdateMySQLConnectionSchema
121+
| UpdateOracleConnectionSchema
122+
| UpdatePostgresConnectionSchema
123+
| UpdateFTPConnectionSchema
124+
| UpdateFTPSConnectionSchema
123125
| UpdateHDFSConnectionSchema
124126
| UpdateS3ConnectionSchema
127+
| UpdateSambaConnectionSchema
125128
| UpdateSFTPConnectionSchema
126-
| UpdateFTPConnectionSchema
127-
| UpdateFTPSConnectionSchema
128-
| UpdateWebDAVConnectionSchema
129-
| UpdateSambaConnectionSchema,
129+
| UpdateWebDAVConnectionSchema,
130130
Field(discriminator="type"),
131131
]
132132

‎syncmaster/schemas/v1/transfers/__init__.py‎

Lines changed: 91 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -10,54 +10,55 @@
1010
from syncmaster.schemas.v1.connections.connection import ReadConnectionSchema
1111
from syncmaster.schemas.v1.page import PageSchema
1212
from syncmaster.schemas.v1.transfers.db import (
13-
ClickhouseReadTransferSourceAndTarget,
14-
HiveReadTransferSourceAndTarget,
15-
MSSQLReadTransferSourceAndTarget,
16-
MySQLReadTransferSourceAndTarget,
17-
OracleReadTransferSourceAndTarget,
18-
PostgresReadTransferSourceAndTarget,
13+
ClickhouseTransferSourceOrTarget,
14+
HiveTransferSourceOrTarget,
15+
IcebergRESTCatalogS3TransferSourceOrTarget,
16+
MSSQLTransferSourceOrTarget,
17+
MySQLTransferSourceOrTarget,
18+
OracleTransferSourceOrTarget,
19+
PostgresTransferSourceOrTarget,
1920
)
2021
from syncmaster.schemas.v1.transfers.file.ftp import (
21-
FTPCreateTransferSource,
22-
FTPCreateTransferTarget,
23-
FTPReadTransferSource,
24-
FTPReadTransferTarget,
22+
CreateFTPTransferSource,
23+
CreateFTPTransferTarget,
24+
ReadFTPTransferSource,
25+
ReadFTPTransferTarget,
2526
)
2627
from syncmaster.schemas.v1.transfers.file.ftps import (
27-
FTPSCreateTransferSource,
28-
FTPSCreateTransferTarget,
29-
FTPSReadTransferSource,
30-
FTPSReadTransferTarget,
28+
CreateFTPSTransferSource,
29+
CreateFTPSTransferTarget,
30+
ReadFTPSTransferSource,
31+
ReadFTPSTransferTarget,
3132
)
3233
from syncmaster.schemas.v1.transfers.file.hdfs import (
33-
HDFSCreateTransferSource,
34-
HDFSCreateTransferTarget,
35-
HDFSReadTransferSource,
36-
HDFSReadTransferTarget,
34+
CreateHDFSTransferSource,
35+
CreateHDFSTransferTarget,
36+
ReadHDFSTransferSource,
37+
ReadHDFSTransferTarget,
3738
)
3839
from syncmaster.schemas.v1.transfers.file.s3 import (
39-
S3CreateTransferSource,
40-
S3CreateTransferTarget,
41-
S3ReadTransferSource,
42-
S3ReadTransferTarget,
40+
CreateS3TransferSource,
41+
CreateS3TransferTarget,
42+
ReadS3TransferSource,
43+
ReadS3TransferTarget,
4344
)
4445
from syncmaster.schemas.v1.transfers.file.samba import (
45-
SambaCreateTransferSource,
46-
SambaCreateTransferTarget,
47-
SambaReadTransferSource,
48-
SambaReadTransferTarget,
46+
CreateSambaTransferSource,
47+
CreateSambaTransferTarget,
48+
ReadSambaTransferSource,
49+
ReadSambaTransferTarget,
4950
)
5051
from syncmaster.schemas.v1.transfers.file.sftp import (
51-
SFTPCreateTransferSource,
52-
SFTPCreateTransferTarget,
53-
SFTPReadTransferSource,
54-
SFTPReadTransferTarget,
52+
CreateSFTPTransferSource,
53+
CreateSFTPTransferTarget,
54+
ReadSFTPTransferSource,
55+
ReadSFTPTransferTarget,
5556
)
5657
from syncmaster.schemas.v1.transfers.file.webdav import (
57-
WebDAVCreateTransferSource,
58-
WebDAVCreateTransferTarget,
59-
WebDAVReadTransferSource,
60-
WebDAVReadTransferTarget,
58+
CreateWebDAVTransferSource,
59+
CreateWebDAVTransferTarget,
60+
ReadWebDAVTransferSource,
61+
ReadWebDAVTransferTarget,
6162
)
6263
from syncmaster.schemas.v1.transfers.resources import Resources
6364
from syncmaster.schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy
@@ -73,67 +74,71 @@
7374
from syncmaster.schemas.v1.types import NameConstr
7475

7576
ReadTransferSchemaSource = (
76-
PostgresReadTransferSourceAndTarget
77-
| HiveReadTransferSourceAndTarget
78-
| OracleReadTransferSourceAndTarget
79-
| ClickhouseReadTransferSourceAndTarget
80-
| MSSQLReadTransferSourceAndTarget
81-
| MySQLReadTransferSourceAndTarget
82-
| HDFSReadTransferSource
83-
| S3ReadTransferSource
84-
| SFTPReadTransferSource
85-
| FTPReadTransferSource
86-
| FTPSReadTransferSource
87-
| WebDAVReadTransferSource
88-
| SambaReadTransferSource
77+
ClickhouseTransferSourceOrTarget
78+
| HiveTransferSourceOrTarget
79+
| IcebergRESTCatalogS3TransferSourceOrTarget
80+
| MSSQLTransferSourceOrTarget
81+
| MySQLTransferSourceOrTarget
82+
| OracleTransferSourceOrTarget
83+
| PostgresTransferSourceOrTarget
84+
| ReadHDFSTransferSource
85+
| ReadS3TransferSource
86+
| ReadSFTPTransferSource
87+
| ReadFTPTransferSource
88+
| ReadFTPSTransferSource
89+
| ReadWebDAVTransferSource
90+
| ReadSambaTransferSource
8991
)
9092

9193
ReadTransferSchemaTarget = (
92-
PostgresReadTransferSourceAndTarget
93-
| HiveReadTransferSourceAndTarget
94-
| OracleReadTransferSourceAndTarget
95-
| ClickhouseReadTransferSourceAndTarget
96-
| MSSQLReadTransferSourceAndTarget
97-
| MySQLReadTransferSourceAndTarget
98-
| HDFSReadTransferTarget
99-
| S3ReadTransferTarget
100-
| SFTPReadTransferTarget
101-
| FTPReadTransferTarget
102-
| FTPSReadTransferTarget
103-
| WebDAVReadTransferTarget
104-
| SambaReadTransferTarget
94+
ClickhouseTransferSourceOrTarget
95+
| HiveTransferSourceOrTarget
96+
| IcebergRESTCatalogS3TransferSourceOrTarget
97+
| MSSQLTransferSourceOrTarget
98+
| MySQLTransferSourceOrTarget
99+
| OracleTransferSourceOrTarget
100+
| PostgresTransferSourceOrTarget
101+
| ReadHDFSTransferTarget
102+
| ReadS3TransferTarget
103+
| ReadSFTPTransferTarget
104+
| ReadFTPTransferTarget
105+
| ReadFTPSTransferTarget
106+
| ReadWebDAVTransferTarget
107+
| ReadSambaTransferTarget
105108
)
106109

107110
CreateTransferSchemaSource = (
108-
PostgresReadTransferSourceAndTarget
109-
| HiveReadTransferSourceAndTarget
110-
| OracleReadTransferSourceAndTarget
111-
| ClickhouseReadTransferSourceAndTarget
112-
| MSSQLReadTransferSourceAndTarget
113-
| MySQLReadTransferSourceAndTarget
114-
| HDFSCreateTransferSource
115-
| S3CreateTransferSource
116-
| SFTPCreateTransferSource
117-
| FTPCreateTransferSource
118-
| FTPSCreateTransferSource
119-
| WebDAVCreateTransferSource
120-
| SambaCreateTransferSource
111+
ClickhouseTransferSourceOrTarget
112+
| HiveTransferSourceOrTarget
113+
| IcebergRESTCatalogS3TransferSourceOrTarget
114+
| MSSQLTransferSourceOrTarget
115+
| MySQLTransferSourceOrTarget
116+
| OracleTransferSourceOrTarget
117+
| PostgresTransferSourceOrTarget
118+
| CreateHDFSTransferSource
119+
| CreateS3TransferSource
120+
| CreateSFTPTransferSource
121+
| CreateFTPTransferSource
122+
| CreateFTPSTransferSource
123+
| CreateWebDAVTransferSource
124+
| CreateSambaTransferSource
121125
)
122126

123127
CreateTransferSchemaTarget = (
124-
PostgresReadTransferSourceAndTarget
125-
| HiveReadTransferSourceAndTarget
126-
| OracleReadTransferSourceAndTarget
127-
| ClickhouseReadTransferSourceAndTarget
128-
| MSSQLReadTransferSourceAndTarget
129-
| MySQLReadTransferSourceAndTarget
130-
| HDFSCreateTransferTarget
131-
| S3CreateTransferTarget
132-
| SFTPCreateTransferTarget
133-
| FTPCreateTransferTarget
134-
| FTPSCreateTransferTarget
135-
| WebDAVCreateTransferTarget
136-
| SambaCreateTransferTarget
128+
ClickhouseTransferSourceOrTarget
129+
| HiveTransferSourceOrTarget
130+
| IcebergRESTCatalogS3TransferSourceOrTarget
131+
| MSSQLTransferSourceOrTarget
132+
| MySQLTransferSourceOrTarget
133+
| OracleTransferSourceOrTarget
134+
| PostgresTransferSourceOrTarget
135+
| CreateHDFSTransferTarget
136+
| CreateS3TransferTarget
137+
| CreateSFTPTransferTarget
138+
| CreateFTPTransferTarget
139+
| CreateFTPSTransferTarget
140+
| CreateWebDAVTransferTarget
141+
| CreateSambaTransferTarget
137142
)
138143

139144
TransformationSchema = DataframeRowsFilter | DataframeColumnsFilter | FileMetadataFilter
@@ -185,17 +190,14 @@ class CreateTransferSchema(BaseModel):
185190
queue_id: int = Field(description="id of the queue in which the transfer will be performed")
186191
schedule: str | None = Field(default=None, description="Execution schedule in cron format")
187192
source_params: CreateTransferSchemaSource = Field(
188-
...,
189193
discriminator="type",
190194
description="Data source parameters",
191195
)
192196
target_params: CreateTransferSchemaTarget = Field(
193-
...,
194197
discriminator="type",
195198
description="Data receiver parameters",
196199
)
197200
strategy_params: FullStrategy | IncrementalStrategy = Field(
198-
...,
199201
discriminator="type",
200202
description="Incremental or archive download options",
201203
)

0 commit comments

Comments
 (0)