Introduce option to use the system cert store for TLS authentication.

janosmurai · janosmurai · commit 8ebe3e80b26f · 2024-11-12T15:37:55.000+01:00
Enhance configuration management by adding default options and loading logic for registries, catalogs, hosts, certs and timeout settings. Introduce platform configuration method and update documentation and test cases.
diff --git a/dem/__main__.py b/dem/__main__.py
@@ -25,6 +25,9 @@ def main() -> None:
         # Load the configuration file
         dem.cli.main.platform.config_file.update()
 
+        # Configure the Development Platform
+        dem.cli.main.platform.configure()
+
         # Load the Dev Env descriptors
         dem.cli.main.platform.load_dev_envs()
 
diff --git a/dem/core/data_management.py b/dem/core/data_management.py
@@ -89,23 +89,25 @@ class ConfigFile(BaseJSON):
     def __init__(self) -> None:
         """ Init the class."""
         self._path = PurePath(self._config_dir + "/config.json")
-        self._default_json = """{
-    "registries": [
-        {
-            "name": "axem",
-            "namespace": "axemsolutions",
-            "url": "https://registry.hub.docker.com"
-        }
-    ],
-    "catalogs": [
-        {
-            "name": "axem",
-            "url": "https://axemsolutions.io/dem/dev_env_org.json"
+        self._default_options = {
+            "registries": [        
+                {
+                    "name": "axem",
+                    "namespace": "axemsolutions",
+                    "url": "https://registry.hub.docker.com"
+                }
+            ],
+            "catalogs": [
+                {
+                    "name": "axem",
+                    "url": "https://axemsolutions.io/dem/dev_env_org.json"
+                }
+            ],
+            "hosts": [],
+            "http_request_timeout_s": 2,
+            "use_native_system_cert_store": False
         }
-    ],
-    "hosts": [],
-    "http_request_timeout_s": 2
-}"""
+        self._default_json = json.dumps(self._default_options, indent=4)
         super().__init__()
 
     def update(self) -> None:
@@ -114,10 +116,37 @@ def update(self) -> None:
         except json.decoder.JSONDecodeError as e:
             raise DataStorageError(f"The config.json file is corrupted.\n{str(e)}") from e
 
-        self.registries: list[dict] = self.deserialized.get("registries", [])
-        self.catalogs: list[dict] = self.deserialized.get("catalogs", [])
-        self.hosts: list[dict] = self.deserialized.get("hosts", [])
+        flush_needed = False
+
+        self.registries: list[dict] | None = self.deserialized.get("registries", None)
+        if self.registries is None:
+            self.deserialized["registries"] = self._default_options["registries"]
+            self.registries = self._default_options["registries"]
+            flush_needed = True
+
+        self.catalogs: list[dict] | None = self.deserialized.get("catalogs", None)
+        if self.catalogs is None:
+            self.deserialized["catalogs"] = self._default_options["catalogs"]
+            self.catalogs = self._default_options["catalogs"]
+            flush_needed = True
+
+        self.hosts: list[dict] | None = self.deserialized.get("hosts", None)
+        if self.hosts is None:
+            self.deserialized["hosts"] = self._default_options["hosts"]
+            self.hosts = self._default_options["hosts"]
+            flush_needed = True
+
         self.http_request_timeout_s: float = self.deserialized.get("http_request_timeout_s", None)
-        
         if self.http_request_timeout_s is None:
-            raise DataStorageError("The http_request_timeout_s is not set in the config.json file.")
+            self.deserialized["http_request_timeout_s"] = self._default_options["http_request_timeout_s"]
+            self.http_request_timeout_s = self._default_options["http_request_timeout_s"]
+            flush_needed = True
+
+        self.use_native_system_cert_store: bool | None = self.deserialized.get("use_native_system_cert_store", None)
+        if self.use_native_system_cert_store is None:
+            self.deserialized["use_native_system_cert_store"] = self._default_options["use_native_system_cert_store"]
+            self.use_native_system_cert_store = self._default_options["use_native_system_cert_store"]
+            flush_needed = True
+
+        if flush_needed:
+            self.flush()
diff --git a/dem/core/platform.py b/dem/core/platform.py
@@ -1,7 +1,7 @@
 """Repesents the Development Platform. The platform resources can be accessed through this interface.  
 """
 
-import os
+import os, truststore
 from typing import Any, Generator
 from dem.core.core import Core
 from dem.core.properties import __supported_dev_env_major_version__
@@ -45,6 +45,11 @@ def __init__(self) -> None:
         # Set this to true in the platform instance to get the tool image info from the registries
         self.get_tool_image_info_from_registries = False
 
+    def configure(self) -> None:
+        """ Configure the Development Platform."""
+        if self.config_file.use_native_system_cert_store:
+            truststore.inject_into_ssl()
+
     def load_dev_envs(self) -> None:
         """ Load the Development Environments from the dev_env.json file.
         
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -0,0 +1,94 @@
+## registries
+
+The `registries` section of the configuration file is used to define the registries that DEM will 
+use to search for images. 
+
+**Default value:**
+
+```json
+"registries": [
+    {
+        "name": "axem",
+        "namespace": "axemsolutions",
+        "url": "https://registry.hub.docker.com"
+    }
+]
+```
+
+!!! info
+    The Registry Management commands can be used to manage the registries in the configuration file.  
+
+    - [`add-reg`](commands.md#dem-add-reg-name-url-namespace): Add a new registry to the configuration file.
+    - [`del-reg`](commands.md#dem-del-reg-name): Delete a registry from the configuration file.
+    - [`list-reg`](commands.md#dem-list-reg): List the registries in the configuration file.
+
+
+## catalogs
+
+The `catalogs` section of the configuration file is used to define the catalogs that DEM will use to
+search for Development Environment descriptors.
+
+** Default value: **
+
+```json
+"catalogs": [
+    {
+        "name": "axem",
+        "url": "https://axemsolutions.io/dem/dev_env_org.json"
+    }
+]
+```
+
+!!! info
+    The Catalog Management commands can be used to manage the catalogs in the configuration file.
+
+    - [`add-cat`](commands.md#dem-add-cat-name-url): Add a new catalog to the configuration file.
+    - [`del-cat`](commands.md#dem-del-cat-name): Delete a catalog from the configuration file.
+    - [`list-cat`](commands.md#dem-list-cat): List the catalogs in the configuration file.
+
+## hosts
+
+!!! warning
+    Remote hosts are not yet supported in DEM.
+
+The `hosts` section of the configuration file is used to define the hosts that DEM will use as 
+remote execution environments.
+
+**Default value:**
+
+```json
+"hosts": []
+```
+
+!!! info
+    The Host Management commands can be used to manage the hosts in the configuration file.
+
+    - [`add-host`](commands.md#dem-add-host-name-url): Add a new host to the configuration file.
+    - [`del-host`](commands.md#dem-del-host-name): Delete a host from the configuration file.
+    - [`list-host`](commands.md#dem-list-host): List the hosts in the configuration file.
+
+## http_request_timeout_s
+
+The `http_request_timeout_s` section of the configuration file is used to define the timeout for
+HTTP requests in seconds.
+
+**Default value:**
+
+```json
+"http_request_timeout_s": 2
+```
+
+## use_native_system_cert_store
+
+The `use_native_system_cert_store` section of the configuration file is used to define whether
+the native system certificate store should be used for HTTPS requests or the default one provided
+by the `certifi` package.
+
+**Default value:**
+
+```json
+"use_native_system_cert_store": false
+```
+
+!!! info
+    If the TLS authentication fails, try setting this value to `true`.
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -92,6 +92,7 @@ nav:
   - 'index.md'
   - 'installation.md'
   - 'basics.md'
+  - 'quickstart.md'
   - Commands: 
     - 'DevEnv Management':
       - 'add-task': 'commands/#dem-add-task-dev_env_name-task_name-command'
@@ -125,6 +126,6 @@ nav:
       - 'add-host': 'commands/#dem-add-host-name-address'
       - 'del-host': 'commands/#dem-del-host-name'
       - 'list-host': 'commands/#dem-list-host'
-  - 'quickstart.md'
+  - 'configuration.md'
   - 'design.md'
   - Join us on Discord  : 'https://discord.gg/3aHuJBNvrJ'
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,9 +30,10 @@ dem = "dem.__main__:main"
 python = "^3.10"
 docker = "^7.0.0"
 readchar = "^4.0.6"
-requests = "^2.31.0"
+requests = "2.31.0"
 rich = "^13.7.1"
 typer = "^0.12.2"
+truststore = "^0.10.0"
 
 [tool.poetry.group.docs]
 optional = true
diff --git a/tests/core/test_data_management.py b/tests/core/test_data_management.py
@@ -170,10 +170,6 @@ def test_ConfigFile(mock_PurePath: MagicMock):
     test_path = "test_path"
     data_management.BaseJSON._config_dir = test_path
 
-    mock_registries = MagicMock()
-    mock_catalogs = MagicMock()
-    mock_hosts = MagicMock()
-
     # Run unit under test
     local_dev_env_json = data_management.ConfigFile()
 
@@ -194,13 +190,15 @@ def test_ConfigFile(mock_PurePath: MagicMock):
         }
     ],
     "hosts": [],
-    "http_request_timeout_s": 2
+    "http_request_timeout_s": 2,
+    "use_native_system_cert_store": false
 }"""
 
     mock_PurePath.assert_called_once_with(test_path + "/config.json")
 
+@patch.object(data_management.BaseJSON, "flush")
 @patch.object(data_management.BaseJSON, "update")
-def test_ConfigFile_update(mock_update: MagicMock) -> None:
+def test_ConfigFile_update(mock_update: MagicMock, mock_flush: MagicMock) -> None:
     # Test setup
     test_config_file = data_management.ConfigFile()
     test_registry = MagicMock()
@@ -224,28 +222,7 @@ def test_ConfigFile_update(mock_update: MagicMock) -> None:
     assert test_config_file.http_request_timeout_s == test_http_request_timeout_s
 
     mock_update.assert_called_once()
-
-@patch.object(data_management.BaseJSON, "update")
-def test_ConfigFile_update_missing_http_request_timeout_s(mock_update: MagicMock) -> None:
-    # Test setup
-    test_config_file = data_management.ConfigFile()
-    test_registry = MagicMock()
-    test_catalog = MagicMock()
-    test_host = MagicMock()
-    test_config_file.deserialized = {
-        "registries": [test_registry],
-        "catalogs": [test_catalog],
-        "hosts": [test_host],
-    }
-
-    with pytest.raises(data_management.DataStorageError) as e:
-        # Run unit under test
-        test_config_file.update()
-
-    # Check expectations
-    assert "Invalid file: The http_request_timeout_s is not set in the config.json file." == str(e.value)
-
-    mock_update.assert_called_once()
+    mock_flush.assert_called_once()
 
 @patch.object(data_management.BaseJSON, "update")
 def test_ConfigFile_update_JSONDecodeError(mock_update: MagicMock) -> None:
