Pass certificates explicitly (#222)

Author: thenno

app/dl_control_api/dl_control_api/app_factory.py

@@ -36,6 +36,7 @@ def _get_env_manager_factory(self, settings: ControlApiAppSettings) -> EnvManage
     def _get_inst_specific_sr_factory(
         self,
         settings: ControlApiAppSettings,
+        ca_data: bytes,
     ) -> StandaloneServiceRegistryFactory:
         return StandaloneServiceRegistryFactory()
 

app/dl_data_api/dl_data_api/app_factory.py

@@ -17,6 +17,7 @@
 from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig
 from dl_configs.connectors_settings import ConnectorSettingsBase
 from dl_configs.enums import RequiredService
+from dl_configs.utils import get_root_certificates
 from dl_constants.enums import ConnectionType
 from dl_core.aio.middlewares.services_registry import services_registry_middleware
 from dl_core.aio.middlewares.us_manager import service_us_manager_middleware
@@ -43,6 +44,7 @@ def _get_env_manager_factory(self, settings: AppSettings) -> EnvManagerFactory:
     def _get_inst_specific_sr_factory(
         self,
         settings: AppSettings,
+        ca_data: bytes,
     ) -> Optional[InstallationSpecificServiceRegistryFactory]:
         return None
 
@@ -78,9 +80,14 @@ def set_up_environment(
         sr_middleware_list: list[AIOHTTPMiddleware]
         usm_middleware_list: list[AIOHTTPMiddleware]
 
+        ca_data = get_root_certificates(path=self._settings.CA_FILE_PATH)
+
         conn_opts_factory = ConnOptionsMutatorsFactory()
         sr_factory = self.get_sr_factory(
-            settings=self._settings, conn_opts_factory=conn_opts_factory, connectors_settings=connectors_settings
+            settings=self._settings,
+            conn_opts_factory=conn_opts_factory,
+            connectors_settings=connectors_settings,
+            ca_data=ca_data,
         )
 
         # Auth middlewares
@@ -105,6 +112,7 @@ def set_up_environment(
         common_us_kw = dict(
             us_base_url=self._settings.US_BASE_URL,
             crypto_keys_config=self._settings.CRYPTO_KEYS_CONFIG,
+            ca_data=ca_data,
         )
         usm_middleware_list = [
             service_us_manager_middleware(us_master_token=self._settings.US_MASTER_TOKEN, **common_us_kw),  # type: ignore

lib/dl_api_commons/dl_api_commons/aiohttp/aiohttp_client.py

@@ -22,8 +22,6 @@
 import aiohttp.web
 import attr
 
-from dl_configs.utils import get_root_certificates_path
-
 
 LOGGER = logging.getLogger(__name__)
 
@@ -93,10 +91,6 @@ async def retry_request(
         raise Exception("You should not be here")
 
 
-def default_ssl_context() -> ssl.SSLContext:
-    return ssl.create_default_context(cafile=get_root_certificates_path())
-
-
 @attr.s(kw_only=True)
 class BIAioHTTPClient:
     base_url: str = attr.ib()
@@ -112,26 +106,24 @@ class BIAioHTTPClient:
 
     retrier: BaseRetrier = attr.ib(factory=NoRetriesRetrier)
 
-    ssl_context: Optional[ssl.SSLContext] = attr.ib(factory=default_ssl_context)
-    _session: Optional[aiohttp.ClientSession] = attr.ib()
-    close_session_on_exit: Optional[bool] = attr.ib(default=None)
+    _ca_data: bytes = attr.ib()
+    _session: Optional[aiohttp.ClientSession] = attr.ib(init=False)
 
-    def make_default_session(self) -> aiohttp.ClientSession:
+    def __attrs_post_init__(self):
+        self._session = self._make_session()
+
+    def _make_session(self) -> aiohttp.ClientSession:
+        ssl_context = ssl.create_default_context(cadata=self._ca_data.decode("utf-8"))
         return aiohttp.ClientSession(
-            cookies=self.cookies, headers=self.headers, connector=aiohttp.TCPConnector(ssl_context=self.ssl_context)
+            cookies=self.cookies,
+            headers=self.headers,
+            connector=aiohttp.TCPConnector(
+                ssl_context=ssl_context,
+            ),
         )
 
-    @property
-    def session(self) -> aiohttp.ClientSession:
-        if self._session is None:
-            self._session = self.make_default_session()
-            if self.close_session_on_exit is None:
-                self.close_session_on_exit = True
-        return self._session
-
     async def close(self) -> None:
-        if self.close_session_on_exit:
-            await self.session.close()
+        await self._session.close()
 
     async def __aenter__(self) -> BIAioHTTPClient:
         return self
@@ -168,7 +160,7 @@ async def _request(
             sock_connect=conn_timeout_sec or self.conn_timeout_sec,
             sock_read=read_timeout_sec or self.read_timeout_sec,
         )
-        return await self.session.request(
+        return await self._session.request(
             method=method,
             url=self.url(path),
             params=params,

lib/dl_api_commons/dl_api_commons/client/base.py

@@ -1,5 +1,4 @@
 import logging
-import ssl
 from typing import (
     Any,
     AsyncIterable,
@@ -17,7 +16,6 @@
     TenantDef,
 )
 from dl_api_commons.tracing import get_current_tracing_headers
-from dl_configs.utils import get_root_certificates_path
 from dl_constants.api_constants import (
     DLHeaders,
     DLHeadersCommon,
@@ -57,21 +55,15 @@ def __init__(self, msg: str, content: bytes, status: int, request_id: str):
         self.request_id = request_id
 
 
-def get_default_aiohttp_session() -> aiohttp.ClientSession:
-    return aiohttp.ClientSession(
-        connector=aiohttp.TCPConnector(ssl_context=ssl.create_default_context(cafile=get_root_certificates_path())),
-    )
-
-
 @attr.s(auto_attribs=True)
 class DLCommonAPIClient:
     _base_url: str = attr.ib()
     _tenant: TenantDef = attr.ib()
     _auth_data: AuthData = attr.ib()
+    _session: aiohttp.ClientSession = attr.ib()
     _req_id: Optional[str] = attr.ib(default=None)
 
     _extra_headers: Optional[dict[DLHeaders, str]] = attr.ib(default=None)
-    _session: aiohttp.ClientSession = attr.ib(factory=get_default_aiohttp_session)
 
     @staticmethod
     def update_dl_headers(acc: dict[DLHeaders, str], update: dict[DLHeaders, str], stage: str) -> None:

lib/dl_api_lib/dl_api_lib/app/control_api/app.py

@@ -34,6 +34,7 @@
     ControlApiAppTestingsSettings,
 )
 from dl_configs.connectors_settings import ConnectorSettingsBase
+from dl_configs.utils import get_root_certificates
 from dl_constants.enums import (
     ConnectionType,
     USAuthMode,
@@ -103,6 +104,8 @@ def create_app(
         ).wrap_flask_app(app)
         ContextVarMiddleware().wrap_flask_app(app)
 
+        ca_data = get_root_certificates(path=self._settings.CA_FILE_PATH)
+
         if close_loop_after_request:
             AIOEventLoopMiddleware().wrap_flask_app(app)
 
@@ -135,6 +138,7 @@ def create_app(
             settings=self._settings,
             conn_opts_factory=conn_opts_mutators_factory,
             connectors_settings=connectors_settings,
+            ca_data=get_root_certificates(self._settings.CA_FILE_PATH),
         )
 
         ServicesRegistryMiddleware(
@@ -146,6 +150,7 @@ def create_app(
             us_base_url=self._settings.US_BASE_URL,
             us_master_token=self._settings.US_MASTER_TOKEN,
             us_auth_mode=env_setup_result.us_auth_mode,
+            ca_data=ca_data,
         ).set_up(app)
 
         app.logger

lib/dl_api_lib/dl_api_lib/app_common.py

@@ -103,6 +103,7 @@ def _get_env_manager_factory(self, settings: TSettings) -> EnvManagerFactory:
     def _get_inst_specific_sr_factory(
         self,
         settings: TSettings,
+        ca_data: bytes,
     ) -> Optional[InstallationSpecificServiceRegistryFactory]:
         raise NotImplementedError
 
@@ -136,6 +137,7 @@ def get_sr_factory(
         settings: TSettings,
         conn_opts_factory: ConnOptionsMutatorsFactory,
         connectors_settings: dict[ConnectionType, ConnectorSettingsBase],
+        ca_data: bytes,
     ) -> DefaultApiSRFactory:
         supported_functions_manager = SupportedFunctionsManager(supported_tags=settings.FORMULA_SUPPORTED_FUNC_TAGS)
 
@@ -181,9 +183,13 @@ def get_sr_factory(
             localizer_factory=localization_factory,
             localizer_fallback=localizer_fallback,
             connector_availability=self._get_connector_availability(settings),
-            inst_specific_sr_factory=self._get_inst_specific_sr_factory(settings),
+            inst_specific_sr_factory=self._get_inst_specific_sr_factory(
+                settings,
+                ca_data=ca_data,
+            ),
             force_non_rqe_mode=settings.RQE_FORCE_OFF,
             query_proc_mode=settings.QUERY_PROCESSING_MODE,
+            ca_data=ca_data,
             pivot_transformer_factory=pivot_transformer_factory,
         )
         return sr_factory

lib/dl_api_lib/dl_api_lib/app_settings.py

@@ -21,7 +21,10 @@
 )
 from dl_configs.settings_loaders.settings_obj_base import SettingsBase
 from dl_configs.settings_submodels import RedisSettings
-from dl_configs.utils import split_by_comma
+from dl_configs.utils import (
+    get_root_certificates_path,
+    split_by_comma,
+)
 from dl_constants.enums import (
     DataPivotEngineType,
     QueryProcessingMode,
@@ -141,6 +144,7 @@ class AppSettings:
         env_var_converter=lambda s: QueryProcessingMode[s.lower()],
         missing=QueryProcessingMode.basic,
     )
+    CA_FILE_PATH: str = s_attrib("CA_FILE_PATH", missing=get_root_certificates_path())
 
     PIVOT_ENGINE_TYPE: DataPivotEngineType = s_attrib(  # type: ignore
         "PIVOT_ENGINE_TYPE",
@@ -234,6 +238,8 @@ class DataApiAppSettings(AppSettings):
 
     BI_ASYNC_APP_DISABLE_KEEPALIVE: bool = s_attrib("BI_ASYNC_APP_DISABLE_KEEPALIVE", missing=False)  # type: ignore
 
+    CA_FILE_PATH: str = s_attrib("CA_FILE_PATH", missing=get_root_certificates_path())
+
     @property
     def app_name(self) -> str:
         return "bi_data_api"

lib/dl_api_lib_testing/dl_api_lib_testing/app.py

@@ -53,6 +53,7 @@
 from dl_core.utils import FutureRef
 from dl_core_testing.app_test_workarounds import TestEnvManagerFactory
 from dl_core_testing.fixture_server_runner import WSGIRunner
+from dl_testing.utils import get_root_certificates
 
 
 @attr.s
@@ -152,6 +153,7 @@ def _get_env_manager_factory(self, settings: AppSettings) -> EnvManagerFactory:
     def _get_inst_specific_sr_factory(
         self,
         settings: AppSettings,
+        ca_data: bytes,
     ) -> Optional[InstallationSpecificServiceRegistryFactory]:
         return TestingServiceRegistryFactory()
 
@@ -209,8 +211,13 @@ def set_up_environment(
         connectors_settings: dict[ConnectionType, ConnectorSettingsBase],
     ) -> DataApiEnvSetupResult:
         conn_opts_factory = ConnOptionsMutatorsFactory()
+        ca_data = get_root_certificates()
+
         sr_factory = self.get_sr_factory(
-            settings=self._settings, conn_opts_factory=conn_opts_factory, connectors_settings=connectors_settings
+            settings=self._settings,
+            conn_opts_factory=conn_opts_factory,
+            connectors_settings=connectors_settings,
+            ca_data=ca_data,
         )
 
         auth_mw_list = [
@@ -232,6 +239,7 @@ def set_up_environment(
         common_us_kw = dict(
             us_base_url=self._settings.US_BASE_URL,
             crypto_keys_config=self._settings.CRYPTO_KEYS_CONFIG,
+            ca_data=ca_data,
         )
         usm_middleware_list = [
             service_us_manager_middleware(us_master_token=self._settings.US_MASTER_TOKEN, **common_us_kw),  # type: ignore[arg-type]

lib/dl_api_lib_testing/dl_api_lib_testing/base.py

@@ -44,6 +44,10 @@
     FlaskTestClient,
     FlaskTestResponse,
 )
+from dl_testing.utils import (
+    get_root_certificates,
+    get_root_certificates_path,
+)
 
 
 class ApiTestBase(abc.ABC):
@@ -115,6 +119,7 @@ def create_control_api_settings(
             FILE_UPLOADER_BASE_URL="http://127.0.0.1:9999",  # fake url
             FILE_UPLOADER_MASTER_TOKEN="qwerty",
             QUERY_PROCESSING_MODE=cls.query_processing_mode,
+            CA_FILE_PATH=get_root_certificates_path(),
         )
         return settings
 
@@ -129,6 +134,10 @@ def control_api_app_settings(
             rqe_config_subprocess=rqe_config_subprocess,
         )
 
+    @pytest.fixture(scope="function")
+    def ca_data(self) -> bytes:
+        return get_root_certificates()
+
     @pytest.fixture(scope="function")
     def control_api_app_factory(self, control_api_app_settings: ControlApiAppSettings) -> ControlApiAppFactory:
         return TestingControlApiAppFactory(settings=control_api_app_settings)
@@ -183,6 +192,7 @@ def sync_us_manager(
         control_api_app_factory: ControlApiAppFactory,
         connectors_settings: dict[ConnectionType, ConnectorSettingsBase],
         control_api_app_settings: ControlApiAppSettings,
+        ca_data: bytes,
     ) -> SyncUSManager:
         core_test_config = bi_test_config.core_test_config
         bi_context = RequestContextInfo.create_empty()
@@ -194,6 +204,7 @@ def sync_us_manager(
                 conn_opts_factory=ConnOptionsMutatorsFactory(),
                 connectors_settings=connectors_settings,
                 settings=control_api_app_settings,
+                ca_data=ca_data,
             ).make_service_registry(request_context_info=bi_context),
             us_base_url=us_config.us_host,
             us_auth_context=USAuthContextMaster(us_config.us_master_token),

lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py

@@ -34,6 +34,7 @@
 from dl_constants.enums import ConnectionType
 from dl_core.components.ids import FieldIdGeneratorType
 from dl_core_testing.database import DbTable
+from dl_testing.utils import get_root_certificates_path
 
 
 class DataApiTestParams(NamedTuple):
@@ -85,6 +86,7 @@ def create_data_api_settings(
             FILE_UPLOADER_BASE_URL=f"{bi_test_config.file_uploader_api_host}:{bi_test_config.file_uploader_api_port}",
             FILE_UPLOADER_MASTER_TOKEN="qwerty",
             QUERY_PROCESSING_MODE=cls.query_processing_mode,
+            CA_FILE_PATH=get_root_certificates_path(),
         )  # type: ignore
 
     @pytest.fixture(scope="function")