Skip to content

Commit b7697ca

Browse files
author
Ilyas Gasanov
committed
[DOP-29495] Add Iceberg OAuth2ClientCredentials
1 parent 9d2124b commit b7697ca

File tree

11 files changed

+378
-62
lines changed

11 files changed

+378
-62
lines changed

‎docs/changelog/next_release/294.feature.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added OAuth2ClientCredentials to Iceberg REST Catalog

‎poetry.lock‎

Lines changed: 107 additions & 43 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎syncmaster/schemas/v1/auth/__init__.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,6 @@
66
ReadBasicAuthSchema,
77
UpdateBasicAuthSchema,
88
)
9-
from syncmaster.schemas.v1.auth.iceberg_rest_basic import (
10-
CreateIcebergRESTCatalogBasicAuthSchema,
11-
IcebergRESTCatalogBasicAuthSchema,
12-
ReadIcebergRESTCatalogBasicAuthSchema,
13-
UpdateIcebergRESTCatalogBasicAuthSchema,
14-
)
159
from syncmaster.schemas.v1.auth.s3 import (
1610
CreateS3AuthSchema,
1711
ReadS3AuthSchema,
@@ -41,8 +35,4 @@
4135
"UpdateSambaAuthSchema",
4236
"AuthTokenSchema",
4337
"TokenPayloadSchema",
44-
"IcebergRESTCatalogBasicAuthSchema",
45-
"CreateIcebergRESTCatalogBasicAuthSchema",
46-
"ReadIcebergRESTCatalogBasicAuthSchema",
47-
"UpdateIcebergRESTCatalogBasicAuthSchema",
4838
]

‎syncmaster/schemas/v1/auth/iceberg/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
2+
# SPDX-License-Identifier: Apache-2.0

‎syncmaster/schemas/v1/auth/iceberg/auth.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
2+
# SPDX-License-Identifier: Apache-2.0
3+
from typing import Annotated
4+
5+
from pydantic import Field
6+
7+
from syncmaster.schemas.v1.auth.iceberg.basic import (
8+
CreateIcebergRESTCatalogBasicAuthSchema,
9+
ReadIcebergRESTCatalogBasicAuthSchema,
10+
UpdateIcebergRESTCatalogBasicAuthSchema,
11+
)
12+
from syncmaster.schemas.v1.auth.iceberg.oauth2_client_credentials import (
13+
CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
14+
ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
15+
UpdateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
16+
)
17+
18+
CreateIcebergRESTCatalogS3ConnectionAuthDataSchema = Annotated[
19+
CreateIcebergRESTCatalogBasicAuthSchema | CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
20+
Field(discriminator="type"),
21+
]
22+
23+
ReadIcebergRESTCatalogS3ConnectionAuthDataSchema = Annotated[
24+
ReadIcebergRESTCatalogBasicAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
25+
Field(discriminator="type"),
26+
]
27+
28+
UpdateIcebergRESTCatalogS3ConnectionAuthDataSchema = Annotated[
29+
UpdateIcebergRESTCatalogBasicAuthSchema | UpdateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
30+
Field(discriminator="type"),
31+
]

syncmaster/schemas/v1/auth/iceberg_rest_basic.py renamed to syncmaster/schemas/v1/auth/iceberg/basic.py

File renamed without changes.

‎syncmaster/schemas/v1/auth/iceberg/oauth2_client_credentials.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
2+
# SPDX-License-Identifier: Apache-2.0
3+
from typing import Literal
4+
5+
from pydantic import BaseModel, Field, SecretStr
6+
7+
8+
class IcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(BaseModel):
9+
type: Literal["iceberg_rest_oauth2_client_credentials_s3_basic"]
10+
11+
12+
class CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(IcebergRESTCatalogOAuth2ClientCredentialsAuthSchema):
13+
metastore_oauth2_client_id: str
14+
metastore_oauth2_client_secret: SecretStr
15+
metastore_oauth2_scopes: list[str] = Field(default_factory=list)
16+
metastore_oauth2_resource: str | None = None
17+
metastore_oauth2_audience: str | None = None
18+
metastore_oauth2_server_uri: str | None = None
19+
s3_access_key: str
20+
s3_secret_key: SecretStr
21+
22+
23+
class ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(IcebergRESTCatalogOAuth2ClientCredentialsAuthSchema):
24+
metastore_oauth2_client_id: str
25+
metastore_oauth2_scopes: list[str]
26+
metastore_oauth2_resource: str | None
27+
metastore_oauth2_audience: str | None
28+
metastore_oauth2_server_uri: str | None
29+
s3_access_key: str
30+
31+
32+
class UpdateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(
33+
CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
34+
):
35+
metastore_oauth2_client_secret: SecretStr | None = None
36+
s3_secret_key: SecretStr | None = None
37+
38+
def get_secret_fields(self) -> tuple[str, ...]:
39+
return ("metastore_password", "s3_secret_key")

‎syncmaster/schemas/v1/connections/connection_base.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,23 @@
44

55
from syncmaster.schemas.v1.auth import (
66
ReadBasicAuthSchema,
7-
ReadIcebergRESTCatalogBasicAuthSchema,
87
ReadS3AuthSchema,
98
ReadSambaAuthSchema,
109
)
10+
from syncmaster.schemas.v1.auth.iceberg.basic import (
11+
ReadIcebergRESTCatalogBasicAuthSchema,
12+
)
13+
from syncmaster.schemas.v1.auth.iceberg.oauth2_client_credentials import (
14+
ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
15+
)
1116
from syncmaster.schemas.v1.types import NameConstr
1217

1318
ReadConnectionAuthDataSchema = (
14-
ReadBasicAuthSchema | ReadS3AuthSchema | ReadSambaAuthSchema | ReadIcebergRESTCatalogBasicAuthSchema
19+
ReadBasicAuthSchema
20+
| ReadS3AuthSchema
21+
| ReadSambaAuthSchema
22+
| ReadIcebergRESTCatalogBasicAuthSchema
23+
| ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema
1524
)
1625

1726

‎syncmaster/schemas/v1/connections/iceberg.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55

66
from pydantic import BaseModel, Field
77

8-
from syncmaster.schemas.v1.auth.iceberg_rest_basic import (
9-
CreateIcebergRESTCatalogBasicAuthSchema,
10-
ReadIcebergRESTCatalogBasicAuthSchema,
11-
UpdateIcebergRESTCatalogBasicAuthSchema,
8+
from syncmaster.schemas.v1.auth.iceberg.auth import (
9+
CreateIcebergRESTCatalogS3ConnectionAuthDataSchema,
10+
ReadIcebergRESTCatalogS3ConnectionAuthDataSchema,
11+
UpdateIcebergRESTCatalogS3ConnectionAuthDataSchema,
1212
)
1313
from syncmaster.schemas.v1.connection_types import ICEBERG_REST_S3_TYPE
1414
from syncmaster.schemas.v1.connections.connection_base import (
@@ -50,18 +50,18 @@ class CreateIcebergConnectionSchema(CreateConnectionBaseSchema):
5050
"Data required to connect to the database. These are the parameters that are specified in the URL request."
5151
),
5252
)
53-
auth_data: CreateIcebergRESTCatalogBasicAuthSchema = Field(
53+
auth_data: CreateIcebergRESTCatalogS3ConnectionAuthDataSchema = Field(
5454
description="Credentials for authorization",
5555
)
5656

5757

5858
class ReadIcebergConnectionSchema(ReadConnectionBaseSchema):
5959
type: ICEBERG_REST_S3_TYPE
6060
data: ReadIcebergRESTCatalogS3ConnectionDataSchema = Field(alias="connection_data")
61-
auth_data: ReadIcebergRESTCatalogBasicAuthSchema | None = None
61+
auth_data: ReadIcebergRESTCatalogS3ConnectionAuthDataSchema | None = None
6262

6363

6464
class UpdateIcebergConnectionSchema(CreateIcebergConnectionSchema):
65-
auth_data: UpdateIcebergRESTCatalogBasicAuthSchema = Field(
65+
auth_data: UpdateIcebergRESTCatalogS3ConnectionAuthDataSchema = Field(
6666
description="Credentials for authorization",
6767
)

‎tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,89 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection(
8888
"s3_access_key": decrypted["s3_access_key"],
8989
},
9090
}
91+
92+
93+
async def test_developer_plus_can_create_iceberg_rest_s3_connection_with_oauth2_client_credentials(
94+
client: AsyncClient,
95+
group: MockGroup,
96+
session: AsyncSession,
97+
settings: Settings,
98+
role_developer_plus: UserTestRoles,
99+
):
100+
user = group.get_member_of_role(role_developer_plus)
101+
102+
result = await client.post(
103+
"v1/connections",
104+
headers={"Authorization": f"Bearer {user.token}"},
105+
json={
106+
"group_id": group.id,
107+
"name": "New connection",
108+
"description": "",
109+
"type": "iceberg_rest_s3",
110+
"connection_data": {
111+
"metastore_url": "https://rest.domain.com",
112+
"s3_warehouse_path": "/some/warehouse",
113+
"s3_protocol": "http",
114+
"s3_host": "localhost",
115+
"s3_port": 9010,
116+
"s3_bucket": "some_bucket",
117+
"s3_region": "us-east-1",
118+
"s3_bucket_style": "path",
119+
},
120+
"auth_data": {
121+
"type": "iceberg_rest_oauth2_client_credentials_s3_basic",
122+
"metastore_oauth2_client_id": "my_client_id",
123+
"metastore_oauth2_client_secret": "my_client_secret",
124+
"metastore_oauth2_scopes": ["catalog:read"],
125+
"metastore_oauth2_audience": "iceberg-catalog",
126+
"metastore_oauth2_server_uri": "https://oauth.example.com/token",
127+
"s3_access_key": "access_key",
128+
"s3_secret_key": "secret_key",
129+
},
130+
},
131+
)
132+
connection = (
133+
await session.scalars(
134+
select(Connection).filter_by(
135+
name="New connection",
136+
),
137+
)
138+
).first()
139+
140+
creds = (
141+
await session.scalars(
142+
select(AuthData).filter_by(
143+
connection_id=connection.id,
144+
),
145+
)
146+
).one()
147+
148+
decrypted = decrypt_auth_data(creds.value, settings=settings)
149+
assert result.status_code == 200, result.json()
150+
assert result.json() == {
151+
"id": connection.id,
152+
"group_id": connection.group_id,
153+
"name": connection.name,
154+
"description": connection.description,
155+
"type": connection.type,
156+
"connection_data": {
157+
"metastore_url": connection.data["metastore_url"],
158+
"s3_warehouse_path": connection.data["s3_warehouse_path"],
159+
"s3_protocol": connection.data["s3_protocol"],
160+
"s3_host": connection.data["s3_host"],
161+
"s3_port": connection.data["s3_port"],
162+
"s3_bucket": connection.data["s3_bucket"],
163+
"s3_region": connection.data["s3_region"],
164+
"s3_bucket_style": connection.data["s3_bucket_style"],
165+
"s3_additional_params": connection.data["s3_additional_params"],
166+
},
167+
"auth_data": {
168+
"type": decrypted["type"],
169+
"metastore_oauth2_client_id": decrypted["metastore_oauth2_client_id"],
170+
"metastore_oauth2_scopes": decrypted["metastore_oauth2_scopes"],
171+
"metastore_oauth2_audience": decrypted["metastore_oauth2_audience"],
172+
"metastore_oauth2_resource": decrypted["metastore_oauth2_resource"],
173+
"metastore_oauth2_server_uri": decrypted["metastore_oauth2_server_uri"],
174+
"s3_access_key": decrypted["s3_access_key"],
175+
},
176+
}

0 commit comments

Comments
 (0)