diff --git a/mapillary_tools/api_v4.py b/mapillary_tools/api_v4.py index 90b2e475c..a664ca993 100644 --- a/mapillary_tools/api_v4.py +++ b/mapillary_tools/api_v4.py @@ -1,8 +1,12 @@ +import logging import os +import ssl import typing as T import requests +from requests.adapters import HTTPAdapter +LOG = logging.getLogger(__name__) MAPILLARY_CLIENT_TOKEN = os.getenv( "MAPILLARY_CLIENT_TOKEN", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282" ) @@ -10,10 +14,96 @@ "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com" ) REQUESTS_TIMEOUT = 60 # 1 minutes +USE_SYSTEM_CERTS: bool = False + + +class HTTPSystemCertsAdapter(HTTPAdapter): + """ + This adapter uses the system's certificate store instead of the certifi module. + + The implementation is based on the project https://pypi.org/project/pip-system-certs/, + which has a system-wide effect. + """ + + def init_poolmanager(self, *args, **kwargs): + ssl_context = ssl.create_default_context() + ssl_context.load_default_certs() + kwargs["ssl_context"] = ssl_context + + super().init_poolmanager(*args, **kwargs) + + def cert_verify(self, *args, **kwargs): + super().cert_verify(*args, **kwargs) + + # By default Python requests uses the ca_certs from the certifi module + # But we want to use the certificate store instead. + # By clearing the ca_certs variable we force it to fall back on that behaviour (handled in urllib3) + if "conn" in kwargs: + conn = kwargs["conn"] + else: + conn = args[0] + + conn.ca_certs = None + + +def request_post( + url: str, + data: T.Optional[T.Any] = None, + json: T.Optional[dict] = None, + **kwargs, +) -> requests.Response: + global USE_SYSTEM_CERTS + + if USE_SYSTEM_CERTS: + with requests.Session() as session: + session.mount("https://", HTTPSystemCertsAdapter()) + return session.post(url, data=data, json=json, **kwargs) + + else: + try: + return requests.post(url, data=data, json=json, **kwargs) + except requests.exceptions.SSLError as ex: + if "SSLCertVerificationError" not in str(ex): + raise ex + USE_SYSTEM_CERTS = True + # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)'))) + LOG.warning( + "SSL error occurred, falling back to system SSL certificates: %s", ex + ) + with requests.Session() as session: + session.mount("https://", HTTPSystemCertsAdapter()) + return session.post(url, data=data, json=json, **kwargs) + + +def request_get( + url: str, + params: T.Optional[dict] = None, + **kwargs, +) -> requests.Response: + global USE_SYSTEM_CERTS + + if USE_SYSTEM_CERTS: + with requests.Session() as session: + session.mount("https://", HTTPSystemCertsAdapter()) + return session.get(url, params=params, **kwargs) + else: + try: + return requests.get(url, params=params, **kwargs) + except requests.exceptions.SSLError as ex: + if "SSLCertVerificationError" not in str(ex): + raise ex + USE_SYSTEM_CERTS = True + # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)'))) + LOG.warning( + "SSL error occurred, falling back to system SSL certificates: %s", ex + ) + with requests.Session() as session: + session.mount("https://", HTTPSystemCertsAdapter()) + return session.get(url, params=params, **kwargs) def get_upload_token(email: str, password: str) -> requests.Response: - resp = requests.post( + resp = request_post( f"{MAPILLARY_GRAPH_API_ENDPOINT}/login", params={"access_token": MAPILLARY_CLIENT_TOKEN}, json={"email": email, "password": password, "locale": "en_US"}, @@ -26,7 +116,7 @@ def get_upload_token(email: str, password: str) -> requests.Response: def fetch_organization( user_access_token: str, organization_id: T.Union[int, str] ) -> requests.Response: - resp = requests.get( + resp = request_get( f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}", params={ "fields": ",".join(["slug", "description", "name"]), @@ -45,8 +135,8 @@ def fetch_organization( ] -def logging(action_type: ActionType, properties: T.Dict) -> requests.Response: - resp = requests.post( +def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response: + resp = request_post( f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging", json={ "action_type": action_type, diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index 4a8904ba4..39a0cc8ea 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -438,7 +438,7 @@ def _api_logging_finished(summary: T.Dict): action: api_v4.ActionType = "upload_finished_upload" LOG.debug("API Logging for action %s: %s", action, summary) try: - api_v4.logging( + api_v4.log_event( action, summary, ) @@ -460,7 +460,7 @@ def _api_logging_failed(payload: T.Dict, exc: Exception): action: api_v4.ActionType = "upload_failed_upload" LOG.debug("API Logging for action %s: %s", action, payload) try: - api_v4.logging( + api_v4.log_event( action, payload_with_reason, ) diff --git a/mapillary_tools/upload_api_v4.py b/mapillary_tools/upload_api_v4.py index 10014340d..fe56e8f8d 100644 --- a/mapillary_tools/upload_api_v4.py +++ b/mapillary_tools/upload_api_v4.py @@ -8,13 +8,12 @@ import requests +from .api_v4 import MAPILLARY_GRAPH_API_ENDPOINT, request_get, request_post + LOG = logging.getLogger(__name__) MAPILLARY_UPLOAD_ENDPOINT = os.getenv( "MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads" ) -MAPILLARY_GRAPH_API_ENDPOINT = os.getenv( - "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com" -) DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16 # 16MB # According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of # (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts @@ -93,7 +92,7 @@ def fetch_offset(self) -> int: } url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}" LOG.debug("GET %s", url) - resp = requests.get( + resp = request_get( url, headers=headers, timeout=REQUESTS_TIMEOUT, @@ -134,7 +133,7 @@ def upload( } url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}" LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers))) - resp = requests.post( + resp = request_post( url, headers=headers, data=chunk, @@ -180,7 +179,7 @@ def finish(self, file_handle: str) -> str: url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload" LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers))) - resp = requests.post( + resp = request_post( url, headers=headers, json=data,