implement server_ca_cert_file config option (#1852)

* implement server_ca_cert_file config option

* update changelog

* revert PEM fingerprint

this was changed during local development and was committed mistakenly

---------

Co-authored-by: Colton Myers <[email protected]>

Author: beniwohli

CHANGELOG.asciidoc

@@ -29,6 +29,17 @@ endif::[]
 //===== Bug fixes
 //
 
+=== Unreleased
+
+// Unreleased changes go here
+// When the next release happens, nest these changes under the "Python Agent version 6.x" heading
+[float]
+==== Features
+ * Add `server_ca_cert_file` option to provide custom CA certificate {pull}1852[#1852]
+
+// [float]
+// ===== Bug fixes
+
 
 [[release-notes-6.x]]
 === Python Agent version 6.x

docs/configuration.asciidoc

@@ -1007,8 +1007,6 @@ By default, the agent verifies the SSL certificate if an HTTPS connection to the
 Verification can be disabled by changing this setting to `False`.
 This setting is ignored when <<config-server-cert,`server_cert`>> is set.
 
-NOTE: SSL certificate verification is only available in Python 2.7.9+ and Python 3.4.3+.
-
 [float]
 [[config-server-cert]]
 ==== `server_cert`
@@ -1019,10 +1017,30 @@ NOTE: SSL certificate verification is only available in Python 2.7.9+ and Python
 | `ELASTIC_APM_SERVER_CERT`  | `SERVER_CERT` | `None`
 |============
 
-If you have configured your APM Server with a self signed TLS certificate, or you
+If you have configured your APM Server with a self-signed TLS certificate, or you
 just wish to pin the server certificate, you can specify the path to the PEM-encoded
 certificate via the `ELASTIC_APM_SERVER_CERT` configuration.
 
+NOTE: If this option is set, the agent only verifies that the certificate provided by the APM Server is
+identical to the one configured here. Validity of the certificate is not checked.
+
+[float]
+[[config-server-ca-cert-file]]
+==== `server_ca_cert_file`
+
+[options="header"]
+|============
+| Environment                        | Django/Flask          | Default
+| `ELASTIC_APM_SERVER_CA_CERT_FILE`  | `SERVER_CA_CERT_FILE` | `None`
+|============
+
+By default, the agent will validate the TLS/SSL certificate of the APM Server using the well-known CAs curated by Mozilla,
+and provided by the https://pypi.org/project/certifi/[`certifi`] package.
+
+You can set this option to the path of a file containing a CA certificate that will be used instead.
+
+Specifying this option is required when using self-signed certificates, unless server certificate validation is disabled.
+
 [float]
 [[config-use-certifi]]
 ==== `use_certifi`

elasticapm/base.py

@@ -149,6 +149,7 @@ def __init__(self, config=None, **inline):
             "headers": headers,
             "verify_server_cert": self.config.verify_server_cert,
             "server_cert": self.config.server_cert,
+            "server_ca_cert_file": self.config.server_ca_cert_file,
             "timeout": self.config.server_timeout,
             "processors": self.load_processors(),
         }

elasticapm/conf/__init__.py

@@ -558,6 +558,7 @@ class Config(_ConfigBase):
     debug = _BoolConfigValue("DEBUG", default=False)
     server_url = _ConfigValue("SERVER_URL", default="http://127.0.0.1:8200", required=True)
     server_cert = _ConfigValue("SERVER_CERT", validators=[FileIsReadableValidator()])
+    server_ca_cert_file = _ConfigValue("SERVER_CA_CERT_FILE", validators=[FileIsReadableValidator()])
     verify_server_cert = _BoolConfigValue("VERIFY_SERVER_CERT", default=True)
     use_certifi = _BoolConfigValue("USE_CERTIFI", default=True)
     include_paths = _ListConfigValue("INCLUDE_PATHS")

elasticapm/transport/http.py

@@ -243,9 +243,11 @@ def auth_headers(self):
     @property
     def ca_certs(self):
         """
-        Return location of certificate store. If it is available and not disabled via setting,
-        this will return the location of the certifi certificate store.
+        Return location of certificate store. If the server_ca_cert_file config option is set,
+        its value is returned. Otherwise, the certifi store is used, unless it is disabled or not installed.
         """
+        if self._server_ca_cert_file:
+            return self._server_ca_cert_file
         return certifi.where() if (certifi and self.client.config.use_certifi) else None
 
 

elasticapm/transport/http_base.py

@@ -44,11 +44,13 @@ def __init__(
         headers=None,
         timeout=None,
         server_cert=None,
+        server_ca_cert_file=None,
         **kwargs
     ):
         self._url = url
         self._verify_server_cert = verify_server_cert
         self._server_cert = server_cert
+        self._server_ca_cert_file = server_ca_cert_file
         self._timeout = timeout
         self._headers = {
             k.encode("ascii") if isinstance(k, str) else k: v.encode("ascii") if isinstance(v, str) else v

tests/transports/test_urllib3.py

@@ -47,6 +47,8 @@
 except ImportError:
     from urllib import parse as urlparse
 
+CUR_DIR = os.path.dirname(os.path.realpath(__file__))
+
 
 @pytest.mark.flaky(reruns=3)  # test is flaky on Windows
 def test_send(waiting_httpserver, elasticapm_client):
@@ -237,10 +239,9 @@ def test_ssl_cert_pinning_http(waiting_httpserver, elasticapm_client):
 @pytest.mark.flaky(reruns=3)  # test is flaky on Windows
 def test_ssl_cert_pinning(waiting_httpsserver, elasticapm_client):
     waiting_httpsserver.serve_content(code=202, content="", headers={"Location": "https://example.com/foo"})
-    cur_dir = os.path.dirname(os.path.realpath(__file__))
     transport = Transport(
         waiting_httpsserver.url,
-        server_cert=os.path.join(cur_dir, "..", "ca/server.pem"),
+        server_cert=os.path.join(CUR_DIR, "..", "ca/server.pem"),
         verify_server_cert=True,
         client=elasticapm_client,
     )
@@ -252,6 +253,19 @@ def test_ssl_cert_pinning(waiting_httpsserver, elasticapm_client):
         transport.close()
 
 
+@pytest.mark.parametrize(
+    "sending_elasticapm_client",
+    [{"server_ca_cert_file": os.path.normpath(os.path.join(CUR_DIR, "..", "ca/ca.crt"))}],
+    indirect=True,
+)
+def test_custom_ca_cert(sending_elasticapm_client):
+    ca_cert = os.path.normpath(os.path.join(CUR_DIR, "..", "ca/ca.crt"))
+    pool = sending_elasticapm_client._transport.http
+    assert pool.connection_pool_kw["ca_certs"] == ca_cert
+    conn = pool.connection_from_url("https://localhost")
+    assert conn.ca_certs == ca_cert
+
+
 @pytest.mark.flaky(reruns=3)  # test is flaky on Windows
 def test_ssl_cert_pinning_fails(waiting_httpsserver, elasticapm_client):
     waiting_httpsserver.serve_content(code=202, content="", headers={"Location": "https://example.com/foo"})

tests/utils/tests.py

@@ -233,7 +233,7 @@ def test_url_to_destination_bad_port():
 def test_read_pem_file():
     with open(os.path.join(os.path.dirname(__file__), "..", "ca", "ca.crt"), mode="rb") as f:
         result = read_pem_file(f)
-        assert result.startswith(b"0\x82\x05{0\x82\x03c\xa0\x03\x02\x01\x02\x02\x14")
+        assert result.startswith(b"0\x82\x05{0\x82\x03c\xa0\x03\x02\x01\x02\x02\x14`c\xd8:\xe7")
 
 
 def test_read_pem_file_chain():