fix: fallback to system SSL certs when certifi fails

mapillary_tools/api_v4.py

@@ -1,19 +1,109 @@
+import logging
 import os
+import ssl
 import typing as T
 
 import requests
+from requests.adapters import HTTPAdapter
 
+LOG = logging.getLogger(__name__)
 MAPILLARY_CLIENT_TOKEN = os.getenv(
     "MAPILLARY_CLIENT_TOKEN", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282"
 )
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
 REQUESTS_TIMEOUT = 60  # 1 minutes
+USE_SYSTEM_CERTS: bool = False
+
+
+class HTTPSystemCertsAdapter(HTTPAdapter):
+    """
+    This adapter uses the system's certificate store instead of the certifi module.
+
+    The implementation is based on the project https://pypi.org/project/pip-system-certs/,
+    which has a system-wide effect.
+    """
+
+    def init_poolmanager(self, *args, **kwargs):
+        ssl_context = ssl.create_default_context()
+        ssl_context.load_default_certs()
+        kwargs["ssl_context"] = ssl_context
+
+        super().init_poolmanager(*args, **kwargs)
+
+    def cert_verify(self, *args, **kwargs):
+        super().cert_verify(*args, **kwargs)
+
+        # By default Python requests uses the ca_certs from the certifi module
+        # But we want to use the certificate store instead.
+        # By clearing the ca_certs variable we force it to fall back on that behaviour (handled in urllib3)
+        if "conn" in kwargs:
+            conn = kwargs["conn"]
+        else:
+            conn = args[0]
+
+        conn.ca_certs = None
+
+
+def request_post(
+    url: str,
+    data: T.Optional[T.Any] = None,
+    json: T.Optional[dict] = None,
+    **kwargs,
+) -> requests.Response:
+    global USE_SYSTEM_CERTS
+
+    if USE_SYSTEM_CERTS:
+        with requests.Session() as session:
+            session.mount("https://", HTTPSystemCertsAdapter())
+            return session.post(url, data=data, json=json, **kwargs)
+
+    else:
+        try:
+            return requests.post(url, data=data, json=json, **kwargs)
+        except requests.exceptions.SSLError as ex:
+            if "SSLCertVerificationError" not in str(ex):
+                raise ex
+            USE_SYSTEM_CERTS = True
+            # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
+            LOG.warning(
+                "SSL error occurred, falling back to system SSL certificates: %s", ex
+            )
+            with requests.Session() as session:
+                session.mount("https://", HTTPSystemCertsAdapter())
+                return session.post(url, data=data, json=json, **kwargs)
+
+
+def request_get(
+    url: str,
+    params: T.Optional[dict] = None,
+    **kwargs,
+) -> requests.Response:
+    global USE_SYSTEM_CERTS
+
+    if USE_SYSTEM_CERTS:
+        with requests.Session() as session:
+            session.mount("https://", HTTPSystemCertsAdapter())
+            return session.get(url, params=params, **kwargs)
+    else:
+        try:
+            return requests.get(url, params=params, **kwargs)
+        except requests.exceptions.SSLError as ex:
+            if "SSLCertVerificationError" not in str(ex):
+                raise ex
+            USE_SYSTEM_CERTS = True
+            # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
+            LOG.warning(
+                "SSL error occurred, falling back to system SSL certificates: %s", ex
+            )
+            with requests.Session() as session:
+                session.mount("https://", HTTPSystemCertsAdapter())
+                return session.get(url, params=params, **kwargs)
 
 
 def get_upload_token(email: str, password: str) -> requests.Response:
-    resp = requests.post(
+    resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
         params={"access_token": MAPILLARY_CLIENT_TOKEN},
         json={"email": email, "password": password, "locale": "en_US"},
@@ -26,7 +116,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
 def fetch_organization(
     user_access_token: str, organization_id: T.Union[int, str]
 ) -> requests.Response:
-    resp = requests.get(
+    resp = request_get(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
         params={
             "fields": ",".join(["slug", "description", "name"]),
@@ -45,8 +135,8 @@ def fetch_organization(
 ]
 
 
-def logging(action_type: ActionType, properties: T.Dict) -> requests.Response:
-    resp = requests.post(
+def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response:
+    resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
         json={
             "action_type": action_type,

mapillary_tools/upload.py

@@ -438,7 +438,7 @@ def _api_logging_finished(summary: T.Dict):
     action: api_v4.ActionType = "upload_finished_upload"
     LOG.debug("API Logging for action %s: %s", action, summary)
     try:
-        api_v4.logging(
+        api_v4.log_event(
             action,
             summary,
         )
@@ -460,7 +460,7 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
     action: api_v4.ActionType = "upload_failed_upload"
     LOG.debug("API Logging for action %s: %s", action, payload)
     try:
-        api_v4.logging(
+        api_v4.log_event(
             action,
             payload_with_reason,
         )

mapillary_tools/upload_api_v4.py

@@ -8,13 +8,12 @@
 
 import requests
 
+from .api_v4 import MAPILLARY_GRAPH_API_ENDPOINT, request_get, request_post
+
 LOG = logging.getLogger(__name__)
 MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
     "MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads"
 )
-MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
-    "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
-)
 DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16  # 16MB
 # According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of
 # (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
@@ -93,7 +92,7 @@ def fetch_offset(self) -> int:
         }
         url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
         LOG.debug("GET %s", url)
-        resp = requests.get(
+        resp = request_get(
             url,
             headers=headers,
             timeout=REQUESTS_TIMEOUT,
@@ -134,7 +133,7 @@ def upload(
             }
             url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
             LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-            resp = requests.post(
+            resp = request_post(
                 url,
                 headers=headers,
                 data=chunk,
@@ -180,7 +179,7 @@ def finish(self, file_handle: str) -> str:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
 
         LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-        resp = requests.post(
+        resp = request_post(
             url,
             headers=headers,
             json=data,