[DOP-30631] Improve naming of Iceberg connection params

dolfinus · dolfinus · commit 24978a8dd5e9 · 2025-11-19T23:40:48.000+03:00
diff --git a/.env.docker.test b/.env.docker.test
@@ -52,10 +52,10 @@ TEST_HIVE_CLUSTER=test-hive
 TEST_HIVE_USER=syncmaster
 TEST_HIVE_PASSWORD=123UsedForTestOnly@!
 
-TEST_ICEBERG_METASTORE_URL_FOR_CONFTEST=http://test-iceberg-rest:8181
-TEST_ICEBERG_METASTORE_URL_FOR_WORKER=http://test-iceberg-rest:8181
-TEST_ICEBERG_METASTORE_USERNAME=syncmaster
-TEST_ICEBERG_METASTORE_PASSWORD=123UsedForTestOnly@!
+TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST=http://test-iceberg-rest:8181
+TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER=http://test-iceberg-rest:8181
+TEST_ICEBERG_REST_CATALOG_USERNAME=syncmaster
+TEST_ICEBERG_REST_CATALOG_PASSWORD=123UsedForTestOnly@!
 TEST_ICEBERG_S3_WAREHOUSE_PATH=/data
 TEST_ICEBERG_S3_REGION=us-east-1
 TEST_ICEBERG_S3_BUCKET_STYLE=path
diff --git a/.env.local.test b/.env.local.test
@@ -52,10 +52,10 @@ export TEST_HIVE_CLUSTER=test-hive
 export TEST_HIVE_USER=syncmaster
 export TEST_HIVE_PASSWORD=123UsedForTestOnly@!
 
-export TEST_ICEBERG_METASTORE_URL_FOR_CONFTEST=http://localhost:8181
-export TEST_ICEBERG_METASTORE_URL_FOR_WORKER=http://test-iceberg-rest:8181
-export TEST_ICEBERG_METASTORE_USERNAME=syncmaster
-export TEST_ICEBERG_METASTORE_PASSWORD=123UsedForTestOnly@!
+export TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST=http://localhost:8181
+export TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER=http://test-iceberg-rest:8181
+export TEST_ICEBERG_REST_CATALOG_USERNAME=syncmaster
+export TEST_ICEBERG_REST_CATALOG_PASSWORD=123UsedForTestOnly@!
 export TEST_ICEBERG_S3_WAREHOUSE_PATH=/data
 export TEST_ICEBERG_S3_REGION=us-east-1
 export TEST_ICEBERG_S3_BUCKET_STYLE=path
diff --git a/.gitignore b/.gitignore
@@ -155,7 +155,7 @@ cython_debug/
 .idea/
 .vscode/
 
-**/metastore_db/
+**/rest_catalog_db/
 **/spark-warehouse/
 cached_jars/
 
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -331,10 +331,10 @@ services:
     environment:
       TZ: UTC
       WITH_HIVE_SERVER: 'false'  # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it
-      HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore
-      HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver
-      HIVE_METASTORE_DB_USER: test_hive
-      HIVE_METASTORE_DB_PASSWORD: test_hive
+      HIVE_REST_CATALOG_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore
+      HIVE_REST_CATALOG_DB_DRIVER: org.postgresql.Driver
+      HIVE_REST_CATALOG_DB_USER: test_hive
+      HIVE_REST_CATALOG_DB_PASSWORD: test_hive
     # writing spark dataframe to s3, sftp, ftp, ftps xml file fails without running hive metastore server
     profiles: [hive, hdfs, s3, sftp, ftp, ftps, samba, webdav, all]
 
diff --git a/syncmaster/dto/connections.py b/syncmaster/dto/connections.py
@@ -75,7 +75,7 @@ class HiveConnectionDTO(ConnectionDTO):
 
 @dataclass
 class IcebergRESTCatalogS3ConnectionBaseDTO(ConnectionDTO):
-    metastore_url: str
+    rest_catalog_url: str
     s3_warehouse_path: str
     s3_host: str
     s3_bucket: str
@@ -91,26 +91,26 @@ class IcebergRESTCatalogS3ConnectionBaseDTO(ConnectionDTO):
 
 @dataclass(kw_only=True)
 class IcebergRESTCatalogBasicAuthS3DTO(IcebergRESTCatalogS3ConnectionBaseDTO):
-    metastore_username: str
-    metastore_password: str
-    metastore_auth_type: Literal["basic"] = "basic"
+    rest_catalog_username: str
+    rest_catalog_password: str
+    rest_catalog_auth_type: Literal["basic"] = "basic"
 
 
 @dataclass(kw_only=True)
 class IcebergRESTCatalogOAuth2ClientCredentialsS3DTO(IcebergRESTCatalogS3ConnectionBaseDTO):
-    metastore_oauth2_client_id: str
-    metastore_oauth2_client_secret: str
-    metastore_oauth2_scopes: list[str]
-    metastore_oauth2_resource: str | None = None
-    metastore_oauth2_audience: str | None = None
-    metastore_oauth2_server_uri: str | None = None
-    metastore_auth_type: Literal["oauth2"] = "oauth2"
+    rest_catalog_oauth2_client_id: str
+    rest_catalog_oauth2_client_secret: str
+    rest_catalog_oauth2_scopes: list[str]
+    rest_catalog_oauth2_resource: str | None = None
+    rest_catalog_oauth2_audience: str | None = None
+    rest_catalog_oauth2_server_uri: str | None = None
+    rest_catalog_auth_type: Literal["oauth2"] = "oauth2"
 
 
 # TODO: should be refactored
 class IcebergRESTCatalogS3ConnectionDTO:
     def __new__(cls, **data):
-        if "metastore_oauth2_client_id" in data:
+        if "rest_catalog_oauth2_client_id" in data:
             return IcebergRESTCatalogOAuth2ClientCredentialsS3DTO(**data)
 
         return IcebergRESTCatalogBasicAuthS3DTO(**data)
diff --git a/syncmaster/schemas/v1/auth/iceberg/basic.py b/syncmaster/schemas/v1/auth/iceberg/basic.py
@@ -7,18 +7,18 @@
 
 class ReadIcebergRESTCatalogBasicAuthSchema(BaseModel):
     type: Literal["iceberg_rest_basic_s3_basic"] = Field(description="Auth type")
-    metastore_username: str
+    rest_catalog_username: str
     s3_access_key: str
 
 
 class CreateIcebergRESTCatalogBasicAuthSchema(ReadIcebergRESTCatalogBasicAuthSchema):
-    metastore_password: SecretStr
+    rest_catalog_password: SecretStr
     s3_secret_key: SecretStr
 
 
 class UpdateIcebergRESTCatalogBasicAuthSchema(ReadIcebergRESTCatalogBasicAuthSchema):
-    metastore_password: SecretStr | None = None
+    rest_catalog_password: SecretStr | None = None
     s3_secret_key: SecretStr | None = None
 
     def get_secret_fields(self) -> tuple[str, ...]:
-        return ("metastore_password", "s3_secret_key")
+        return ("rest_catalog_password", "s3_secret_key")
diff --git a/syncmaster/schemas/v1/auth/iceberg/oauth2_client_credentials.py b/syncmaster/schemas/v1/auth/iceberg/oauth2_client_credentials.py
@@ -9,26 +9,26 @@
 
 class ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(BaseModel):
     type: Literal["iceberg_rest_oauth2_client_credentials_s3_basic"] = Field(description="Auth type")
-    metastore_oauth2_client_id: str
-    metastore_oauth2_scopes: list[str] = Field(default_factory=list)
-    metastore_oauth2_resource: str | None = None
-    metastore_oauth2_audience: str | None = None
-    metastore_oauth2_server_uri: URL | None = None
+    rest_catalog_oauth2_client_id: str
+    rest_catalog_oauth2_scopes: list[str] = Field(default_factory=list)
+    rest_catalog_oauth2_resource: str | None = None
+    rest_catalog_oauth2_audience: str | None = None
+    rest_catalog_oauth2_server_uri: URL | None = None
     s3_access_key: str
 
 
 class CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(
     ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
 ):
-    metastore_oauth2_client_secret: SecretStr
+    rest_catalog_oauth2_client_secret: SecretStr
     s3_secret_key: SecretStr
 
 
 class UpdateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(
     ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema,
 ):
-    metastore_oauth2_client_secret: SecretStr | None = None
+    rest_catalog_oauth2_client_secret: SecretStr | None = None
     s3_secret_key: SecretStr | None = None
 
     def get_secret_fields(self) -> tuple[str, ...]:
-        return ("metastore_oauth2_client_secret", "s3_secret_key")
+        return ("rest_catalog_oauth2_client_secret", "s3_secret_key")
diff --git a/syncmaster/schemas/v1/connections/iceberg.py b/syncmaster/schemas/v1/connections/iceberg.py
@@ -19,7 +19,7 @@
 
 
 class IcebergRESTCatalogS3ConnectionDataSchema(BaseModel):
-    metastore_url: URL
+    rest_catalog_url: URL
     s3_warehouse_path: str
     s3_host: str
     s3_port: int | None = Field(default=None, gt=0, le=65535)  # noqa: WPS432
diff --git a/syncmaster/worker/handlers/db/iceberg.py b/syncmaster/worker/handlers/db/iceberg.py
@@ -32,7 +32,7 @@ def connect(self, spark: SparkSession):
             spark=spark,
             catalog_name=self.transfer_dto.catalog_name,
             catalog=Iceberg.RESTCatalog(
-                uri=self.connection_dto.metastore_url,
+                uri=self.connection_dto.rest_catalog_url,
                 auth=self._make_auth(),
             ),
             warehouse=Iceberg.S3Warehouse(
@@ -90,16 +90,16 @@ def _quote_field(self, field: str) -> str:
         return f"`{field}`"
 
     def _make_auth(self):
-        if self.connection_dto.metastore_auth_type == "oauth2":
+        if self.connection_dto.rest_catalog_auth_type == "oauth2":
             return Iceberg.RESTCatalog.OAuth2ClientCredentials(
-                client_id=self.connection_dto.metastore_oauth2_client_id,
-                client_secret=self.connection_dto.metastore_oauth2_client_secret,
-                scopes=self.connection_dto.metastore_oauth2_scopes,
-                resource=self.connection_dto.metastore_oauth2_resource,
-                audience=self.connection_dto.metastore_oauth2_audience,
-                server_uri=self.connection_dto.metastore_oauth2_server_uri,
+                client_id=self.connection_dto.rest_catalog_oauth2_client_id,
+                client_secret=self.connection_dto.rest_catalog_oauth2_client_secret,
+                scopes=self.connection_dto.rest_catalog_oauth2_scopes,
+                resource=self.connection_dto.rest_catalog_oauth2_resource,
+                audience=self.connection_dto.rest_catalog_oauth2_audience,
+                server_uri=self.connection_dto.rest_catalog_oauth2_server_uri,
             )
         return Iceberg.RESTCatalog.BasicAuth(
-            user=self.connection_dto.metastore_username,
-            password=self.connection_dto.metastore_password,
+            user=self.connection_dto.rest_catalog_username,
+            password=self.connection_dto.rest_catalog_password,
         )
diff --git a/tests/settings.py b/tests/settings.py
@@ -57,10 +57,10 @@ class TestSettings(BaseSettings):
     TEST_HIVE_USER: str
     TEST_HIVE_PASSWORD: str
 
-    TEST_ICEBERG_METASTORE_URL_FOR_CONFTEST: str
-    TEST_ICEBERG_METASTORE_URL_FOR_WORKER: str
-    TEST_ICEBERG_METASTORE_USERNAME: str
-    TEST_ICEBERG_METASTORE_PASSWORD: str
+    TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST: str
+    TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER: str
+    TEST_ICEBERG_REST_CATALOG_USERNAME: str
+    TEST_ICEBERG_REST_CATALOG_PASSWORD: str
     TEST_ICEBERG_S3_WAREHOUSE_PATH: str
     TEST_ICEBERG_S3_REGION: str
     TEST_ICEBERG_S3_BUCKET_STYLE: Literal["domain", "path"] = "path"
diff --git a/tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py b/tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py
@@ -23,7 +23,7 @@
 )
 def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCatalogS3ConnectionDTO:
     return IcebergRESTCatalogS3ConnectionDTO(
-        metastore_url=test_settings.TEST_ICEBERG_METASTORE_URL_FOR_CONFTEST,
+        rest_catalog_url=test_settings.TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST,
         s3_warehouse_path=test_settings.TEST_ICEBERG_S3_WAREHOUSE_PATH,
         s3_host=test_settings.TEST_S3_HOST_FOR_CONFTEST,
         s3_port=test_settings.TEST_S3_PORT_FOR_CONFTEST,
@@ -34,8 +34,8 @@ def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCata
         s3_access_key=test_settings.TEST_S3_ACCESS_KEY,
         s3_secret_key=test_settings.TEST_S3_SECRET_KEY,
         s3_additional_params=test_settings.TEST_ICEBERG_S3_ADDITIONAL_PARAMS,
-        metastore_username=test_settings.TEST_ICEBERG_METASTORE_USERNAME,
-        metastore_password=test_settings.TEST_ICEBERG_METASTORE_PASSWORD,
+        rest_catalog_username=test_settings.TEST_ICEBERG_REST_CATALOG_USERNAME,
+        rest_catalog_password=test_settings.TEST_ICEBERG_REST_CATALOG_PASSWORD,
     )
 
 
@@ -45,7 +45,7 @@ def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCata
 )
 def iceberg_rest_s3_for_worker(test_settings: TestSettings) -> IcebergRESTCatalogS3ConnectionDTO:
     return IcebergRESTCatalogS3ConnectionDTO(
-        metastore_url=test_settings.TEST_ICEBERG_METASTORE_URL_FOR_WORKER,
+        rest_catalog_url=test_settings.TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER,
         s3_warehouse_path=test_settings.TEST_ICEBERG_S3_WAREHOUSE_PATH,
         s3_host=test_settings.TEST_S3_HOST_FOR_WORKER,
         s3_port=test_settings.TEST_S3_PORT_FOR_WORKER,
@@ -56,8 +56,8 @@ def iceberg_rest_s3_for_worker(test_settings: TestSettings) -> IcebergRESTCatalo
         s3_access_key=test_settings.TEST_S3_ACCESS_KEY,
         s3_secret_key=test_settings.TEST_S3_SECRET_KEY,
         s3_additional_params=test_settings.TEST_ICEBERG_S3_ADDITIONAL_PARAMS,
-        metastore_username=test_settings.TEST_ICEBERG_METASTORE_USERNAME,
-        metastore_password=test_settings.TEST_ICEBERG_METASTORE_PASSWORD,
+        rest_catalog_username=test_settings.TEST_ICEBERG_REST_CATALOG_USERNAME,
+        rest_catalog_password=test_settings.TEST_ICEBERG_REST_CATALOG_PASSWORD,
     )
 
 
@@ -77,10 +77,10 @@ def prepare_iceberg_rest_s3(
         spark=spark,
         catalog_name=catalog_name,
         catalog=Iceberg.RESTCatalog(
-            uri=iceberg.metastore_url,
+            uri=iceberg.rest_catalog_url,
             auth=Iceberg.RESTCatalog.BasicAuth(
-                user=iceberg.metastore_username,
-                password=iceberg.metastore_password,
+                user=iceberg.rest_catalog_username,
+                password=iceberg.rest_catalog_password,
             ),
         ),
         warehouse=Iceberg.S3Warehouse(
@@ -129,7 +129,7 @@ async def iceberg_rest_s3_connection(
         name=secrets.token_hex(5),
         type=iceberg.type,
         data=dict(
-            metastore_url=iceberg.metastore_url,
+            rest_catalog_url=iceberg.rest_catalog_url,
             s3_warehouse_path=iceberg.s3_warehouse_path,
             s3_host=iceberg.s3_host,
             s3_port=iceberg.s3_port,
@@ -150,8 +150,8 @@ async def iceberg_rest_s3_connection(
             type="iceberg_rest_basic_s3_basic",
             s3_access_key=iceberg.s3_access_key,
             s3_secret_key=iceberg.s3_secret_key,
-            metastore_username=iceberg.metastore_username,
-            metastore_password=iceberg.metastore_password,
+            rest_catalog_username=iceberg.rest_catalog_username,
+            rest_catalog_password=iceberg.rest_catalog_password,
         ),
     )
 
diff --git a/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py b/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py
@@ -34,7 +34,7 @@ async def group_connections(
         elif conn_type == ConnectionType.ICEBERG_REST_S3:
             new_data.update(
                 {
-                    "metastore_url": "https://rest.domain.com",
+                    "rest_catalog_url": "https://rest.domain.com",
                     "s3_warehouse_path": "/some/warehouse",
                     "s3_host": "s3.domain.com",
                     "s3_bucket": "bucket",
diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py
@@ -29,7 +29,7 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection(
             "description": "",
             "type": "iceberg_rest_s3",
             "connection_data": {
-                "metastore_url": "https://rest.domain.com",
+                "rest_catalog_url": "https://rest.domain.com",
                 "s3_warehouse_path": "/some/warehouse",
                 "s3_protocol": "http",
                 "s3_host": "localhost",
@@ -40,8 +40,8 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection(
             },
             "auth_data": {
                 "type": "iceberg_rest_basic_s3_basic",
-                "metastore_username": "user",
-                "metastore_password": "secret",
+                "rest_catalog_username": "user",
+                "rest_catalog_password": "secret",
                 "s3_access_key": "access_key",
                 "s3_secret_key": "secret_key",
             },
@@ -72,7 +72,7 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection(
         "description": connection.description,
         "type": connection.type,
         "connection_data": {
-            "metastore_url": connection.data["metastore_url"],
+            "rest_catalog_url": connection.data["rest_catalog_url"],
             "s3_warehouse_path": connection.data["s3_warehouse_path"],
             "s3_protocol": connection.data["s3_protocol"],
             "s3_host": connection.data["s3_host"],
@@ -84,7 +84,7 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection(
         },
         "auth_data": {
             "type": decrypted["type"],
-            "metastore_username": decrypted["metastore_username"],
+            "rest_catalog_username": decrypted["rest_catalog_username"],
             "s3_access_key": decrypted["s3_access_key"],
         },
     }
@@ -108,7 +108,7 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection_with_oauth2_
             "description": "",
             "type": "iceberg_rest_s3",
             "connection_data": {
-                "metastore_url": "https://rest.domain.com",
+                "rest_catalog_url": "https://rest.domain.com",
                 "s3_warehouse_path": "/some/warehouse",
                 "s3_protocol": "http",
                 "s3_host": "localhost",
@@ -119,11 +119,11 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection_with_oauth2_
             },
             "auth_data": {
                 "type": "iceberg_rest_oauth2_client_credentials_s3_basic",
-                "metastore_oauth2_client_id": "my_client_id",
-                "metastore_oauth2_client_secret": "my_client_secret",
-                "metastore_oauth2_scopes": ["catalog:read"],
-                "metastore_oauth2_audience": "iceberg-catalog",
-                "metastore_oauth2_server_uri": "https://oauth.example.com/token",
+                "rest_catalog_oauth2_client_id": "my_client_id",
+                "rest_catalog_oauth2_client_secret": "my_client_secret",
+                "rest_catalog_oauth2_scopes": ["catalog:read"],
+                "rest_catalog_oauth2_audience": "iceberg-catalog",
+                "rest_catalog_oauth2_server_uri": "https://oauth.example.com/token",
                 "s3_access_key": "access_key",
                 "s3_secret_key": "secret_key",
             },
@@ -154,7 +154,7 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection_with_oauth2_
         "description": connection.description,
         "type": connection.type,
         "connection_data": {
-            "metastore_url": connection.data["metastore_url"],
+            "rest_catalog_url": connection.data["rest_catalog_url"],
             "s3_warehouse_path": connection.data["s3_warehouse_path"],
             "s3_protocol": connection.data["s3_protocol"],
             "s3_host": connection.data["s3_host"],
@@ -166,11 +166,11 @@ async def test_developer_plus_can_create_iceberg_rest_s3_connection_with_oauth2_
         },
         "auth_data": {
             "type": decrypted["type"],
-            "metastore_oauth2_client_id": decrypted["metastore_oauth2_client_id"],
-            "metastore_oauth2_scopes": decrypted["metastore_oauth2_scopes"],
-            "metastore_oauth2_audience": decrypted["metastore_oauth2_audience"],
-            "metastore_oauth2_resource": decrypted["metastore_oauth2_resource"],
-            "metastore_oauth2_server_uri": decrypted["metastore_oauth2_server_uri"],
+            "rest_catalog_oauth2_client_id": decrypted["rest_catalog_oauth2_client_id"],
+            "rest_catalog_oauth2_scopes": decrypted["rest_catalog_oauth2_scopes"],
+            "rest_catalog_oauth2_audience": decrypted["rest_catalog_oauth2_audience"],
+            "rest_catalog_oauth2_resource": decrypted["rest_catalog_oauth2_resource"],
+            "rest_catalog_oauth2_server_uri": decrypted["rest_catalog_oauth2_server_uri"],
             "s3_access_key": decrypted["s3_access_key"],
         },
     }
diff --git a/tests/test_unit/test_connections/test_db_connection/test_update_iceberg_connection.py b/tests/test_unit/test_connections/test_db_connection/test_update_iceberg_connection.py
diff --git a/tests/test_unit/utils.py b/tests/test_unit/utils.py
diff --git a/tests/utils.py b/tests/utils.py
