Fix S3 and IcebergRestS3 tests

Author: dolfinus

syncmaster/dto/connections.py

@@ -76,18 +76,18 @@ class HiveConnectionDTO(ConnectionDTO):
 @dataclass
 class IcebergRESTCatalogS3ConnectionDTO(ConnectionDTO):
     metastore_url: str
+    metastore_username: str
+    metastore_password: str
     s3_warehouse_path: str
     s3_host: str
     s3_bucket: str
     s3_bucket_style: Literal["domain", "path"]
     s3_region: str
     s3_access_key: str
     s3_secret_key: str
-    metastore_username: str
-    metastore_password: str
     s3_port: int | None
     s3_protocol: str
-    additional_params: dict
+    s3_additional_params: dict
     type: ClassVar[str] = "iceberg_rest_s3"
 
 

syncmaster/schemas/v1/connections/iceberg.py

@@ -26,7 +26,7 @@ class CreateIcebergRESTCatalogS3ConnectionDataSchema(BaseModel):
     s3_bucket: str
     s3_region: str
     s3_bucket_style: Literal["domain", "path"] = "path"
-    additional_params: dict = Field(default_factory=dict)
+    s3_additional_params: dict = Field(default_factory=dict)
 
 
 class ReadIcebergRESTCatalogS3ConnectionDataSchema(BaseModel):
@@ -38,7 +38,7 @@ class ReadIcebergRESTCatalogS3ConnectionDataSchema(BaseModel):
     s3_bucket: str
     s3_region: str
     s3_bucket_style: Literal["domain", "path"] = "path"
-    additional_params: dict = Field(default_factory=dict)
+    s3_additional_params: dict = Field(default_factory=dict)
 
 
 class CreateIcebergConnectionSchema(CreateConnectionBaseSchema):

tests/settings.py

@@ -14,6 +14,7 @@ class TestSettings(BaseSettings):
     TEST_POSTGRES_DB: str
     TEST_POSTGRES_USER: str
     TEST_POSTGRES_PASSWORD: str
+    TEST_POSTGRES_ADDITIONAL_PARAMS: dict = {}
 
     TEST_ORACLE_HOST_FOR_CONFTEST: str
     TEST_ORACLE_PORT_FOR_CONFTEST: int
@@ -23,6 +24,7 @@ class TestSettings(BaseSettings):
     TEST_ORACLE_PASSWORD: str
     TEST_ORACLE_SID: str | None = None
     TEST_ORACLE_SERVICE_NAME: str | None = None
+    TEST_ORACLE_ADDITIONAL_PARAMS: dict = {}
 
     TEST_CLICKHOUSE_HOST_FOR_CONFTEST: str
     TEST_CLICKHOUSE_PORT_FOR_CONFTEST: int
@@ -31,6 +33,7 @@ class TestSettings(BaseSettings):
     TEST_CLICKHOUSE_USER: str
     TEST_CLICKHOUSE_PASSWORD: str
     TEST_CLICKHOUSE_DB: str
+    TEST_CLICKHOUSE_ADDITIONAL_PARAMS: dict = {}
 
     TEST_MSSQL_HOST_FOR_CONFTEST: str
     TEST_MSSQL_PORT_FOR_CONFTEST: int
@@ -39,6 +42,7 @@ class TestSettings(BaseSettings):
     TEST_MSSQL_USER: str
     TEST_MSSQL_PASSWORD: str
     TEST_MSSQL_DB: str
+    TEST_MSSQL_ADDITIONAL_PARAMS: dict = {}
 
     TEST_MYSQL_HOST_FOR_CONFTEST: str
     TEST_MYSQL_PORT_FOR_CONFTEST: int
@@ -47,6 +51,7 @@ class TestSettings(BaseSettings):
     TEST_MYSQL_USER: str
     TEST_MYSQL_PASSWORD: str
     TEST_MYSQL_DB: str
+    TEST_MYSQL_ADDITIONAL_PARAMS: dict = {}
 
     TEST_HIVE_CLUSTER: str
     TEST_HIVE_USER: str
@@ -59,6 +64,7 @@ class TestSettings(BaseSettings):
     TEST_ICEBERG_S3_WAREHOUSE_PATH: str
     TEST_ICEBERG_S3_REGION: str
     TEST_ICEBERG_S3_BUCKET_STYLE: Literal["domain", "path"] = "path"
+    TEST_ICEBERG_S3_ADDITIONAL_PARAMS: dict = {}
 
     TEST_HDFS_HOST: str
     TEST_HDFS_WEBHDFS_PORT: int

tests/test_integration/test_run_transfer/connection_fixtures/clickhouse_fixtures.py

@@ -28,7 +28,7 @@ def clickhouse_for_conftest(test_settings: TestSettings) -> ClickhouseConnection
         user=test_settings.TEST_CLICKHOUSE_USER,
         password=test_settings.TEST_CLICKHOUSE_PASSWORD,
         database_name=test_settings.TEST_CLICKHOUSE_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_CLICKHOUSE_ADDITIONAL_PARAMS,
     )
 
 
@@ -43,7 +43,7 @@ def clickhouse_for_worker(test_settings: TestSettings) -> ClickhouseConnectionDT
         user=test_settings.TEST_CLICKHOUSE_USER,
         password=test_settings.TEST_CLICKHOUSE_PASSWORD,
         database_name=test_settings.TEST_CLICKHOUSE_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_CLICKHOUSE_ADDITIONAL_PARAMS,
     )
 
 
@@ -112,7 +112,7 @@ async def clickhouse_connection(
             host=clickhouse.host,
             port=clickhouse.port,
             database_name=clickhouse.database_name,
-            additional_params={},
+            additional_params=clickhouse.additional_params,
         ),
         group_id=group.id,
     )

tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py

@@ -33,9 +33,9 @@ def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCata
         s3_bucket_style=test_settings.TEST_ICEBERG_S3_BUCKET_STYLE,
         s3_access_key=test_settings.TEST_S3_ACCESS_KEY,
         s3_secret_key=test_settings.TEST_S3_SECRET_KEY,
+        s3_additional_params=test_settings.TEST_ICEBERG_S3_ADDITIONAL_PARAMS,
         metastore_username=test_settings.TEST_ICEBERG_METASTORE_USERNAME,
         metastore_password=test_settings.TEST_ICEBERG_METASTORE_PASSWORD,
-        additional_params={},
     )
 
 
@@ -55,9 +55,9 @@ def iceberg_rest_s3_for_worker(test_settings: TestSettings) -> IcebergRESTCatalo
         s3_bucket_style=test_settings.TEST_ICEBERG_S3_BUCKET_STYLE,
         s3_access_key=test_settings.TEST_S3_ACCESS_KEY,
         s3_secret_key=test_settings.TEST_S3_SECRET_KEY,
+        s3_additional_params=test_settings.TEST_ICEBERG_S3_ADDITIONAL_PARAMS,
         metastore_username=test_settings.TEST_ICEBERG_METASTORE_USERNAME,
         metastore_password=test_settings.TEST_ICEBERG_METASTORE_PASSWORD,
-        additional_params={},
     )
 
 

tests/test_integration/test_run_transfer/connection_fixtures/mssql_fixtures.py

@@ -28,7 +28,7 @@ def mssql_for_conftest(test_settings: TestSettings) -> MSSQLConnectionDTO:
         user=test_settings.TEST_MSSQL_USER,
         password=test_settings.TEST_MSSQL_PASSWORD,
         database_name=test_settings.TEST_MSSQL_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_MSSQL_ADDITIONAL_PARAMS,
     )
 
 
@@ -43,7 +43,7 @@ def mssql_for_worker(test_settings: TestSettings) -> MSSQLConnectionDTO:
         user=test_settings.TEST_MSSQL_USER,
         password=test_settings.TEST_MSSQL_PASSWORD,
         database_name=test_settings.TEST_MSSQL_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_MSSQL_ADDITIONAL_PARAMS,
     )
 
 
@@ -108,7 +108,7 @@ async def mssql_connection(
             host=mssql.host,
             port=mssql.port,
             database_name=mssql.database_name,
-            additional_params={},
+            additional_params=mssql.additional_params,
         ),
         group_id=group.id,
     )

tests/test_integration/test_run_transfer/connection_fixtures/mysql_fixtures.py

@@ -28,7 +28,7 @@ def mysql_for_conftest(test_settings: TestSettings) -> MySQLConnectionDTO:
         user=test_settings.TEST_MYSQL_USER,
         password=test_settings.TEST_MYSQL_PASSWORD,
         database_name=test_settings.TEST_MYSQL_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_MYSQL_ADDITIONAL_PARAMS,
     )
 
 
@@ -43,7 +43,7 @@ def mysql_for_worker(test_settings: TestSettings) -> MySQLConnectionDTO:
         user=test_settings.TEST_MYSQL_USER,
         password=test_settings.TEST_MYSQL_PASSWORD,
         database_name=test_settings.TEST_MYSQL_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_MYSQL_ADDITIONAL_PARAMS,
     )
 
 
@@ -107,7 +107,7 @@ async def mysql_connection(
             host=mysql.host,
             port=mysql.port,
             database_name=mysql.database_name,
-            additional_params={},
+            additional_params=mysql.additional_params,
         ),
         group_id=group.id,
     )

tests/test_integration/test_run_transfer/connection_fixtures/oracle_fixtures.py

@@ -29,7 +29,7 @@ def oracle_for_conftest(test_settings: TestSettings) -> OracleConnectionDTO:
         password=test_settings.TEST_ORACLE_PASSWORD,
         service_name=test_settings.TEST_ORACLE_SERVICE_NAME,
         sid=test_settings.TEST_ORACLE_SID,
-        additional_params={},
+        additional_params=test_settings.TEST_ORACLE_ADDITIONAL_PARAMS,
     )
 
 
@@ -45,7 +45,7 @@ def oracle_for_worker(test_settings: TestSettings) -> OracleConnectionDTO:
         password=test_settings.TEST_ORACLE_PASSWORD,
         service_name=test_settings.TEST_ORACLE_SERVICE_NAME,
         sid=test_settings.TEST_ORACLE_SID,
-        additional_params={},
+        additional_params=test_settings.TEST_ORACLE_ADDITIONAL_PARAMS,
     )
 
 
@@ -112,7 +112,7 @@ async def oracle_connection(
             port=oracle.port,
             sid=oracle.sid,
             service_name=oracle.service_name,
-            additional_params={},
+            additional_params=oracle.additional_params,
         ),
         group_id=group.id,
     )

tests/test_integration/test_run_transfer/connection_fixtures/postgres_fixtures.py

@@ -28,7 +28,7 @@ def postgres_for_conftest(test_settings: TestSettings) -> PostgresConnectionDTO:
         user=test_settings.TEST_POSTGRES_USER,
         password=test_settings.TEST_POSTGRES_PASSWORD,
         database_name=test_settings.TEST_POSTGRES_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_POSTGRES_ADDITIONAL_PARAMS,
     )
 
 
@@ -43,7 +43,7 @@ def postgres_for_worker(test_settings: TestSettings) -> PostgresConnectionDTO:
         user=test_settings.TEST_POSTGRES_USER,
         password=test_settings.TEST_POSTGRES_PASSWORD,
         database_name=test_settings.TEST_POSTGRES_DB,
-        additional_params={},
+        additional_params=test_settings.TEST_POSTGRES_ADDITIONAL_PARAMS,
     )
 
 
@@ -95,7 +95,7 @@ async def postgres_connection(
             host=postgres.host,
             port=postgres.port,
             database_name=postgres.database_name,
-            additional_params={},
+            additional_params=postgres.additional_params,
         ),
         group_id=group.id,
     )

tests/test_unit/test_connections/test_file_connection/test_create_s3_connection.py

@@ -148,6 +148,7 @@ async def test_developer_plus_can_create_s3_connection_auto_generate_port(
             "region": connection.data["region"],
             "protocol": connection.data["protocol"],
             "bucket_style": connection.data["bucket_style"],
+            "additional_params": connection.data["additional_params"],
         },
         "auth_data": {
             "type": decrypted["type"],