Skip to content

Commit 3564a9d

Browse files
author
Ilyas Gasanov
committed
[DOP-29425] Add Iceberg connection to API
1 parent 044f6a7 commit 3564a9d

File tree

14 files changed

+312
-15
lines changed

14 files changed

+312
-15
lines changed

‎docs/changelog/next_release/281.feature.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added Iceberg Connection to API

‎syncmaster/db/models/connection.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
class ConnectionType(StrEnum):
1919
POSTGRES = "postgres"
2020
HIVE = "hive"
21+
ICEBERG = "iceberg_rest_s3"
2122
ORACLE = "oracle"
2223
CLICKHOUSE = "clickhouse"
2324
MSSQL = "mssql"

‎syncmaster/schemas/v1/auth/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@
66
ReadBasicAuthSchema,
77
UpdateBasicAuthSchema,
88
)
9+
from syncmaster.schemas.v1.auth.iceberg_rest_basic import (
10+
CreateIcebergRESTCatalogBasicAuthSchema,
11+
IcebergRESTCatalogBasicAuthSchema,
12+
ReadIcebergRESTCatalogBasicAuthSchema,
13+
UpdateIcebergRESTCatalogBasicAuthSchema,
14+
)
915
from syncmaster.schemas.v1.auth.s3 import (
1016
CreateS3AuthSchema,
1117
ReadS3AuthSchema,
@@ -35,4 +41,8 @@
3541
"UpdateSambaAuthSchema",
3642
"AuthTokenSchema",
3743
"TokenPayloadSchema",
44+
"IcebergRESTCatalogBasicAuthSchema",
45+
"CreateIcebergRESTCatalogBasicAuthSchema",
46+
"ReadIcebergRESTCatalogBasicAuthSchema",
47+
"UpdateIcebergRESTCatalogBasicAuthSchema",
3848
]

‎syncmaster/schemas/v1/auth/basic.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,5 @@ class ReadBasicAuthSchema(BasicAuthSchema):
2121
class UpdateBasicAuthSchema(CreateBasicAuthSchema):
2222
password: SecretStr | None = None
2323

24-
@property
25-
def secret_field(self) -> str:
26-
return "password"
24+
def get_secret_fields(self) -> tuple[str, ...]:
25+
return ("password",)

‎syncmaster/schemas/v1/auth/iceberg_rest_basic.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
2+
# SPDX-License-Identifier: Apache-2.0
3+
from typing import Literal
4+
5+
from pydantic import BaseModel, SecretStr
6+
7+
8+
class IcebergRESTCatalogBasicAuthSchema(BaseModel):
9+
type: Literal["iceberg_rest_basic_s3_basic"]
10+
11+
12+
class CreateIcebergRESTCatalogBasicAuthSchema(IcebergRESTCatalogBasicAuthSchema):
13+
metastore_username: str
14+
metastore_password: SecretStr
15+
s3_access_key: str
16+
s3_secret_key: SecretStr
17+
18+
19+
class ReadIcebergRESTCatalogBasicAuthSchema(IcebergRESTCatalogBasicAuthSchema):
20+
metastore_username: str
21+
s3_access_key: str
22+
23+
24+
class UpdateIcebergRESTCatalogBasicAuthSchema(CreateIcebergRESTCatalogBasicAuthSchema):
25+
metastore_password: SecretStr | None = None
26+
s3_secret_key: SecretStr | None = None
27+
28+
def get_secret_fields(self) -> tuple[str, ...]:
29+
return ("metastore_password", "s3_secret_key")

‎syncmaster/schemas/v1/auth/s3.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,5 @@ class ReadS3AuthSchema(S3AuthSchema):
2121
class UpdateS3AuthSchema(CreateS3AuthSchema):
2222
secret_key: SecretStr | None = None
2323

24-
@property
25-
def secret_field(self) -> str:
26-
return "secret_key"
24+
def get_secret_fields(self) -> tuple[str, ...]:
25+
return ("secret_key",)

‎syncmaster/schemas/v1/auth/samba.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,5 @@ class ReadSambaAuthSchema(SambaAuthSchema):
2323
class UpdateSambaAuthSchema(CreateSambaAuthSchema):
2424
password: SecretStr | None = None
2525

26-
@property
27-
def secret_field(self) -> str:
28-
return "password"
26+
def get_secret_fields(self) -> tuple[str, ...]:
27+
return ("password",)

‎syncmaster/schemas/v1/connection_types.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from typing import Literal
44

55
HIVE_TYPE = Literal["hive"]
6+
ICEBERG_TYPE = Literal["iceberg_rest_s3"]
67
ORACLE_TYPE = Literal["oracle"]
78
POSTGRES_TYPE = Literal["postgres"]
89
CLICKHOUSE_TYPE = Literal["clickhouse"]
@@ -21,6 +22,7 @@
2122
"postgres",
2223
"clickhouse",
2324
"hive",
25+
"iceberg_rest_s3",
2426
"mssql",
2527
"mysql",
2628
"s3",
@@ -32,4 +34,4 @@
3234
"samba",
3335
]
3436
FILE_CONNECTION_TYPES = ["s3", "hdfs", "sftp", "ftp", "ftps", "webdav", "samba"]
35-
DB_CONNECTION_TYPES = ["oracle", "postgres", "clickhouse", "hive", "mssql", "mysql"]
37+
DB_CONNECTION_TYPES = ["oracle", "postgres", "clickhouse", "hive", "iceberg_rest_s3", "mssql", "mysql"]

‎syncmaster/schemas/v1/connections/connection.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@
3030
ReadHiveConnectionSchema,
3131
UpdateHiveConnectionSchema,
3232
)
33+
from syncmaster.schemas.v1.connections.iceberg import (
34+
CreateIcebergConnectionSchema,
35+
ReadIcebergConnectionSchema,
36+
UpdateIcebergConnectionSchema,
37+
)
3338
from syncmaster.schemas.v1.connections.mssql import (
3439
CreateMSSQLConnectionSchema,
3540
ReadMSSQLConnectionSchema,
@@ -80,6 +85,7 @@
8085
| CreateMSSQLConnectionSchema
8186
| CreateClickhouseConnectionSchema
8287
| CreateHiveConnectionSchema
88+
| CreateIcebergConnectionSchema
8389
| CreateHDFSConnectionSchema
8490
| CreateS3ConnectionSchema
8591
| CreateSFTPConnectionSchema
@@ -96,6 +102,7 @@
96102
| ReadMSSQLConnectionSchema
97103
| ReadClickhouseConnectionSchema
98104
| ReadHiveConnectionSchema
105+
| ReadIcebergConnectionSchema
99106
| ReadHDFSConnectionSchema
100107
| ReadS3ConnectionSchema
101108
| ReadSFTPConnectionSchema
@@ -112,6 +119,7 @@
112119
| UpdateMSSQLConnectionSchema
113120
| UpdateClickhouseConnectionSchema
114121
| UpdateHiveConnectionSchema
122+
| UpdateIcebergConnectionSchema
115123
| UpdateHDFSConnectionSchema
116124
| UpdateS3ConnectionSchema
117125
| UpdateSFTPConnectionSchema

‎syncmaster/schemas/v1/connections/iceberg.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
from typing import Literal
5+
6+
from pydantic import BaseModel, Field, computed_field
7+
8+
from syncmaster.schemas.v1.auth.iceberg_rest_basic import (
9+
CreateIcebergRESTCatalogBasicAuthSchema,
10+
ReadIcebergRESTCatalogBasicAuthSchema,
11+
UpdateIcebergRESTCatalogBasicAuthSchema,
12+
)
13+
from syncmaster.schemas.v1.connection_types import ICEBERG_TYPE
14+
from syncmaster.schemas.v1.connections.connection_base import (
15+
CreateConnectionBaseSchema,
16+
ReadConnectionBaseSchema,
17+
)
18+
19+
20+
class CreateIcebergRESTCatalogS3ConnectionDataSchema(BaseModel):
21+
s3_warehouse_path: str
22+
s3_host: str
23+
s3_port: int | None = None
24+
s3_protocol: Literal["http", "https"] = "https"
25+
s3_bucket: str
26+
s3_region: str
27+
s3_path_style_access: bool = False
28+
29+
@computed_field
30+
@property
31+
def metastore_url(self) -> str:
32+
port = f":{self.s3_port}" if self.s3_port else ""
33+
return f"{self.s3_protocol}://{self.s3_host}{port}"
34+
35+
36+
class ReadIcebergRESTCatalogS3ConnectionDataSchema(BaseModel):
37+
metastore_url: str
38+
s3_warehouse_path: str
39+
s3_host: str
40+
s3_port: int | None = None
41+
s3_protocol: Literal["http", "https"] = "https"
42+
s3_bucket: str
43+
s3_region: str
44+
s3_path_style_access: bool = False
45+
46+
47+
class CreateIcebergConnectionSchema(CreateConnectionBaseSchema):
48+
type: ICEBERG_TYPE = Field(description="Connection type")
49+
data: CreateIcebergRESTCatalogS3ConnectionDataSchema = Field(
50+
...,
51+
alias="connection_data",
52+
description=(
53+
"Data required to connect to the database. These are the parameters that are specified in the URL request."
54+
),
55+
)
56+
auth_data: CreateIcebergRESTCatalogBasicAuthSchema = Field(
57+
description="Credentials for authorization",
58+
)
59+
60+
61+
class ReadIcebergConnectionSchema(ReadConnectionBaseSchema):
62+
type: ICEBERG_TYPE
63+
data: ReadIcebergRESTCatalogS3ConnectionDataSchema = Field(alias="connection_data")
64+
auth_data: ReadIcebergRESTCatalogBasicAuthSchema | None = None
65+
66+
67+
class UpdateIcebergConnectionSchema(CreateIcebergConnectionSchema):
68+
auth_data: UpdateIcebergRESTCatalogBasicAuthSchema = Field(
69+
description="Credentials for authorization",
70+
)

0 commit comments

Comments
 (0)