Add support for mTLS authentication with OpAMP (#419)

* opamp: extend client for mTLS

* distro: get mTLS configuration from env vars

* Add documentation

* Aplly feedback from review

Author: xrmx

docs/reference/edot-python/configuration.md

@@ -67,10 +67,31 @@ product:
 
 If the OpAMP server is configured to require authentication set the `ELASTIC_OTEL_OPAMP_HEADERS` environment variable.
 
-```
+```sh
 export ELASTIC_OTEL_OPAMP_HEADERS="Authorization=ApiKey an_api_key"
 ```
 
+### Configure mTLS for Central configuration
+
+```{applies_to}
+serverless: unavailable
+stack: preview 9.1
+product:
+  edot_python: preview 1.10.0
+```
+
+If the OpAMP Central configuration server requires mutual TLS to encrypt data in transit you need to set the following environment variables:
+
+- `ELASTIC_OTEL_OPAMP_CERTIFICATE`: The path of the trusted certificate to use when verifying a server’s TLS credentials, this may also be used if the server is using a self-signed certificate.
+- `ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE`: Client certificate/chain trust for clients private key path to use in mTLS communication in PEM format.
+- `ELASTIC_OTEL_OPAMP_CLIENT_KEY`: Client private key path to use in mTLS communication in PEM format.
+
+```sh
+export ELASTIC_OTEL_OPAMP_CERTIFICATE=/path/to/rootCA.pem
+export ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE=/path/to/client.pem
+export ELASTIC_OTEL_OPAMP_CLIENT_KEY=/path/to/client-key.pem
+```
+
 ### Central configuration settings
 
 You can modify the following settings for EDOT Python through APM Agent Central Configuration:

src/elasticotel/distro/__init__.py

@@ -60,6 +60,9 @@
     ELASTIC_OTEL_OPAMP_ENDPOINT,
     ELASTIC_OTEL_OPAMP_HEADERS,
     ELASTIC_OTEL_SYSTEM_METRICS_ENABLED,
+    ELASTIC_OTEL_OPAMP_CERTIFICATE,
+    ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE,
+    ELASTIC_OTEL_OPAMP_CLIENT_KEY,
 )
 from elasticotel.distro.resource_detectors import get_cloud_resource_detectors
 from elasticotel.distro.config import opamp_handler, _initialize_config, DEFAULT_SAMPLING_RATE
@@ -129,10 +132,17 @@ def _configure(self, **kwargs):
                 else:
                     headers = None
 
+                # If string is a path to the certificate, if bool means to check the server certificate. Behaviour inherited from requests
+                tls_certificate: str | bool = os.environ.get(ELASTIC_OTEL_OPAMP_CERTIFICATE, True)
+                tls_client_certificate: str | None = os.environ.get(ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE)
+                tls_client_key: str | None = os.environ.get(ELASTIC_OTEL_OPAMP_CLIENT_KEY)
                 opamp_client = OpAMPClient(
                     endpoint=endpoint_url,
                     agent_identifying_attributes=agent_identifying_attributes,
                     headers=headers,
+                    tls_certificate=tls_certificate,
+                    tls_client_certificate=tls_client_certificate,
+                    tls_client_key=tls_client_key,
                 )
                 opamp_agent = OpAMPAgent(
                     interval=30,

src/elasticotel/distro/environment_variables.py

@@ -40,3 +40,30 @@
 
 **Default value:** ``not set``
 """
+
+ELASTIC_OTEL_OPAMP_CERTIFICATE = "ELASTIC_OTEL_OPAMP_CERTIFICATE"
+"""
+.. envvar:: ELASTIC_OTEL_OPAMP_CERTIFICATE
+
+The path of the trusted certificate to use when verifying a server’s TLS credentials, this is needed for mTLS or when the server is using a self-signed certificate.
+
+**Default value:** ``not set``
+"""
+
+ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE = "ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE"
+"""
+.. envvar:: ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE
+
+Client certificate/chain trust for clients private key path to use in mTLS communication in PEM format.
+
+**Default value:** ``not set``
+"""
+
+ELASTIC_OTEL_OPAMP_CLIENT_KEY = "ELASTIC_OTEL_OPAMP_CLIENT_KEY"
+"""
+.. envvar:: ELASTIC_OTEL_OPAMP_CLIENT_KEY
+
+Client private key path to use in mTLS communication in PEM format.
+
+**Default value:** ``not set``
+"""

src/opentelemetry/_opamp/client.py

@@ -60,13 +60,20 @@ def __init__(
         timeout_millis: int = _DEFAULT_OPAMP_TIMEOUT_MS,
         agent_identifying_attributes: Mapping[str, AnyValue],
         agent_non_identifying_attributes: Mapping[str, AnyValue] | None = None,
+        # this matches requests but can be mapped to other http libraries APIs
+        tls_certificate: str | bool = True,
+        tls_client_certificate: str | None = None,
+        tls_client_key: str | None = None,
     ):
         self._timeout_millis = timeout_millis
         self._transport = RequestsTransport()
 
         self._endpoint = endpoint
         headers = headers or {}
         self._headers = {**_OPAMP_HTTP_HEADERS, **headers}
+        self._tls_certificate = tls_certificate
+        self._tls_client_certificate = tls_client_certificate
+        self._tls_client_key = tls_client_key
 
         self._agent_description = messages._build_agent_description(
             identifying_attributes=agent_identifying_attributes,
@@ -154,7 +161,13 @@ def _send(self, data: bytes):
         token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
         try:
             response = self._transport.send(
-                url=self._endpoint, headers=self._headers, data=data, timeout_millis=self._timeout_millis
+                url=self._endpoint,
+                headers=self._headers,
+                data=data,
+                timeout_millis=self._timeout_millis,
+                tls_certificate=self._tls_certificate,
+                tls_client_certificate=self._tls_client_certificate,
+                tls_client_key=self._tls_client_key,
             )
             return response
         finally:

src/opentelemetry/_opamp/transport/base.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import annotations
+
 import abc
 from typing import Mapping
 
@@ -27,5 +29,15 @@
 
 class HttpTransport(abc.ABC):
     @abc.abstractmethod
-    def send(self, url: str, headers: Mapping[str, str], data: bytes, timeout_millis: int) -> opamp_pb2.ServerToAgent:
+    def send(
+        self,
+        *,
+        url: str,
+        headers: Mapping[str, str],
+        data: bytes,
+        timeout_millis: int,
+        tls_certificate: str | bool,
+        tls_client_certificate: str | None = None,
+        tls_client_key: str | None = None,
+    ) -> opamp_pb2.ServerToAgent:
         pass

src/opentelemetry/_opamp/transport/requests.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import annotations
+
 import logging
 from typing import Mapping
 
@@ -32,11 +34,28 @@ def __init__(self):
         self.session = requests.Session()
 
     # TODO: support basic-auth?
-    def send(self, url: str, headers: Mapping[str, str], data: bytes, timeout_millis: int):
+    def send(
+        self,
+        *,
+        url: str,
+        headers: Mapping[str, str],
+        data: bytes,
+        timeout_millis: int,
+        tls_certificate: str | bool,
+        tls_client_certificate: str | None = None,
+        tls_client_key: str | None = None,
+    ):
         headers = {**base_headers, **headers}
         timeout: float = timeout_millis / 1e3
+        client_cert = (
+            (tls_client_certificate, tls_client_key)
+            if tls_client_certificate and tls_client_key
+            else tls_client_certificate
+        )
         try:
-            response = self.session.post(url, headers=headers, data=data, timeout=timeout)
+            response = self.session.post(
+                url, headers=headers, data=data, timeout=timeout, verify=tls_certificate, cert=client_cert
+            )
             response.raise_for_status()
         except Exception as exc:
             logger.error(str(exc))

tests/distro/test_distro.py

@@ -21,7 +21,13 @@
 
 from elasticotel.distro import ElasticOpenTelemetryConfigurator, ElasticOpenTelemetryDistro, logger as distro_logger
 from elasticotel.distro.config import opamp_handler, logger as config_logger, Config
-from elasticotel.distro.environment_variables import ELASTIC_OTEL_OPAMP_ENDPOINT, ELASTIC_OTEL_SYSTEM_METRICS_ENABLED
+from elasticotel.distro.environment_variables import (
+    ELASTIC_OTEL_OPAMP_ENDPOINT,
+    ELASTIC_OTEL_OPAMP_CERTIFICATE,
+    ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE,
+    ELASTIC_OTEL_OPAMP_CLIENT_KEY,
+    ELASTIC_OTEL_SYSTEM_METRICS_ENABLED,
+)
 from elasticotel.sdk.sampler import DefaultSampler
 from opentelemetry.environment_variables import (
     OTEL_LOGS_EXPORTER,
@@ -162,6 +168,9 @@ def test_configurator_sets_up_opamp_with_http_endpoint(self, client_mock, agent_
             endpoint="http://localhost:4320/v1/opamp",
             agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"},
             headers=None,
+            tls_certificate=True,
+            tls_client_certificate=None,
+            tls_client_key=None,
         )
         agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock)
         agent_mock.start.assert_called_once_with()
@@ -186,6 +195,9 @@ def test_configurator_sets_up_opamp_with_https_endpoint(self, client_mock, agent
             endpoint="https://localhost:4320/v1/opamp",
             agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"},
             headers=None,
+            tls_certificate=True,
+            tls_client_certificate=None,
+            tls_client_key=None,
         )
         agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock)
         agent_mock.start.assert_called_once_with()
@@ -211,6 +223,9 @@ def test_configurator_sets_up_opamp_with_headers_from_environment_variable(self,
             endpoint="http://localhost:4320/v1/opamp",
             agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"},
             headers={"authorization": "ApiKey foobar==="},
+            tls_certificate=True,
+            tls_client_certificate=None,
+            tls_client_key=None,
         )
         agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock)
         agent_mock.start.assert_called_once_with()
@@ -235,6 +250,9 @@ def test_configurator_adds_path_to_opamp_endpoint_if_missing(self, client_mock,
             endpoint="https://localhost:4320/v1/opamp",
             agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"},
             headers=None,
+            tls_certificate=True,
+            tls_client_certificate=None,
+            tls_client_key=None,
         )
         agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock)
         agent_mock.start.assert_called_once_with()
@@ -259,6 +277,39 @@ def test_configurator_sets_up_opamp_without_deployment_environment_name(self, cl
             endpoint="https://localhost:4320/v1/opamp",
             agent_identifying_attributes={"service.name": "service"},
             headers=None,
+            tls_certificate=True,
+            tls_client_certificate=None,
+            tls_client_key=None,
+        )
+        agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock)
+        agent_mock.start.assert_called_once_with()
+
+    @mock.patch.dict(
+        "os.environ",
+        {
+            ELASTIC_OTEL_OPAMP_ENDPOINT: "https://localhost:4320/v1/opamp",
+            ELASTIC_OTEL_OPAMP_CERTIFICATE: "server.pem",
+            ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE: "client.pem",
+            ELASTIC_OTEL_OPAMP_CLIENT_KEY: "client.key",
+            "OTEL_RESOURCE_ATTRIBUTES": "service.name=service",
+        },
+        clear=True,
+    )
+    @mock.patch("elasticotel.distro.OpAMPAgent")
+    @mock.patch("elasticotel.distro.OpAMPClient")
+    def test_configurator_sets_up_opamp_with_mTLS_variables(self, client_mock, agent_mock):
+        client_mock.return_value = client_mock
+        agent_mock.return_value = agent_mock
+
+        ElasticOpenTelemetryConfigurator()._configure()
+
+        client_mock.assert_called_once_with(
+            endpoint="https://localhost:4320/v1/opamp",
+            agent_identifying_attributes={"service.name": "service"},
+            headers=None,
+            tls_certificate="server.pem",
+            tls_client_certificate="client.pem",
+            tls_client_key="client.key",
         )
         agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock)
         agent_mock.start.assert_called_once_with()

tests/opamp/test_client.py

@@ -88,6 +88,9 @@ def test_client_headers_override_defaults():
         headers={"Content-Type": "application/x-protobuf", "User-Agent": "Custom"},
         data=b"",
         timeout_millis=1000,
+        tls_certificate=True,
+        tls_client_certificate=None,
+        tls_client_key=None,
     )
 
 
@@ -347,6 +350,9 @@ def test_send(client):
         headers={"Content-Type": "application/x-protobuf", "User-Agent": "OTel-OpAMP-Python/" + __version__},
         data=b"foo",
         timeout_millis=1000,
+        tls_certificate=True,
+        tls_client_certificate=None,
+        tls_client_key=None,
     )