Skip to content

Commit d095726

Browse files
ptptptpt
authored
fix: fallback to system SSL certs when certifi fails (#698)
* add HTTPSystemCertsAdapter * check if it's SSL verification error * usort
1 parent e5165ef commit d095726

File tree

3 files changed

+101
-12
lines changed

3 files changed

+101
-12
lines changed

‎mapillary_tools/api_v4.py‎

Lines changed: 94 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,109 @@
1+
import logging
12
import os
3+
import ssl
24
import typing as T
35

46
import requests
7+
from requests.adapters import HTTPAdapter
58

9+
LOG = logging.getLogger(__name__)
610
MAPILLARY_CLIENT_TOKEN = os.getenv(
711
"MAPILLARY_CLIENT_TOKEN", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282"
812
)
913
MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
1014
"MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
1115
)
1216
REQUESTS_TIMEOUT = 60 # 1 minutes
17+
USE_SYSTEM_CERTS: bool = False
18+
19+
20+
class HTTPSystemCertsAdapter(HTTPAdapter):
21+
"""
22+
This adapter uses the system's certificate store instead of the certifi module.
23+
24+
The implementation is based on the project https://pypi.org/project/pip-system-certs/,
25+
which has a system-wide effect.
26+
"""
27+
28+
def init_poolmanager(self, *args, **kwargs):
29+
ssl_context = ssl.create_default_context()
30+
ssl_context.load_default_certs()
31+
kwargs["ssl_context"] = ssl_context
32+
33+
super().init_poolmanager(*args, **kwargs)
34+
35+
def cert_verify(self, *args, **kwargs):
36+
super().cert_verify(*args, **kwargs)
37+
38+
# By default Python requests uses the ca_certs from the certifi module
39+
# But we want to use the certificate store instead.
40+
# By clearing the ca_certs variable we force it to fall back on that behaviour (handled in urllib3)
41+
if "conn" in kwargs:
42+
conn = kwargs["conn"]
43+
else:
44+
conn = args[0]
45+
46+
conn.ca_certs = None
47+
48+
49+
def request_post(
50+
url: str,
51+
data: T.Optional[T.Any] = None,
52+
json: T.Optional[dict] = None,
53+
**kwargs,
54+
) -> requests.Response:
55+
global USE_SYSTEM_CERTS
56+
57+
if USE_SYSTEM_CERTS:
58+
with requests.Session() as session:
59+
session.mount("https://", HTTPSystemCertsAdapter())
60+
return session.post(url, data=data, json=json, **kwargs)
61+
62+
else:
63+
try:
64+
return requests.post(url, data=data, json=json, **kwargs)
65+
except requests.exceptions.SSLError as ex:
66+
if "SSLCertVerificationError" not in str(ex):
67+
raise ex
68+
USE_SYSTEM_CERTS = True
69+
# HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
70+
LOG.warning(
71+
"SSL error occurred, falling back to system SSL certificates: %s", ex
72+
)
73+
with requests.Session() as session:
74+
session.mount("https://", HTTPSystemCertsAdapter())
75+
return session.post(url, data=data, json=json, **kwargs)
76+
77+
78+
def request_get(
79+
url: str,
80+
params: T.Optional[dict] = None,
81+
**kwargs,
82+
) -> requests.Response:
83+
global USE_SYSTEM_CERTS
84+
85+
if USE_SYSTEM_CERTS:
86+
with requests.Session() as session:
87+
session.mount("https://", HTTPSystemCertsAdapter())
88+
return session.get(url, params=params, **kwargs)
89+
else:
90+
try:
91+
return requests.get(url, params=params, **kwargs)
92+
except requests.exceptions.SSLError as ex:
93+
if "SSLCertVerificationError" not in str(ex):
94+
raise ex
95+
USE_SYSTEM_CERTS = True
96+
# HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
97+
LOG.warning(
98+
"SSL error occurred, falling back to system SSL certificates: %s", ex
99+
)
100+
with requests.Session() as session:
101+
session.mount("https://", HTTPSystemCertsAdapter())
102+
return session.get(url, params=params, **kwargs)
13103

14104

15105
def get_upload_token(email: str, password: str) -> requests.Response:
16-
resp = requests.post(
106+
resp = request_post(
17107
f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
18108
params={"access_token": MAPILLARY_CLIENT_TOKEN},
19109
json={"email": email, "password": password, "locale": "en_US"},
@@ -26,7 +116,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
26116
def fetch_organization(
27117
user_access_token: str, organization_id: T.Union[int, str]
28118
) -> requests.Response:
29-
resp = requests.get(
119+
resp = request_get(
30120
f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
31121
params={
32122
"fields": ",".join(["slug", "description", "name"]),
@@ -45,8 +135,8 @@ def fetch_organization(
45135
]
46136

47137

48-
def logging(action_type: ActionType, properties: T.Dict) -> requests.Response:
49-
resp = requests.post(
138+
def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response:
139+
resp = request_post(
50140
f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
51141
json={
52142
"action_type": action_type,

‎mapillary_tools/upload.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ def _api_logging_finished(summary: T.Dict):
438438
action: api_v4.ActionType = "upload_finished_upload"
439439
LOG.debug("API Logging for action %s: %s", action, summary)
440440
try:
441-
api_v4.logging(
441+
api_v4.log_event(
442442
action,
443443
summary,
444444
)
@@ -460,7 +460,7 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
460460
action: api_v4.ActionType = "upload_failed_upload"
461461
LOG.debug("API Logging for action %s: %s", action, payload)
462462
try:
463-
api_v4.logging(
463+
api_v4.log_event(
464464
action,
465465
payload_with_reason,
466466
)

‎mapillary_tools/upload_api_v4.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@
88

99
import requests
1010

11+
from .api_v4 import MAPILLARY_GRAPH_API_ENDPOINT, request_get, request_post
12+
1113
LOG = logging.getLogger(__name__)
1214
MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
1315
"MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads"
1416
)
15-
MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
16-
"MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
17-
)
1817
DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16 # 16MB
1918
# According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of
2019
# (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
@@ -93,7 +92,7 @@ def fetch_offset(self) -> int:
9392
}
9493
url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
9594
LOG.debug("GET %s", url)
96-
resp = requests.get(
95+
resp = request_get(
9796
url,
9897
headers=headers,
9998
timeout=REQUESTS_TIMEOUT,
@@ -134,7 +133,7 @@ def upload(
134133
}
135134
url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
136135
LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
137-
resp = requests.post(
136+
resp = request_post(
138137
url,
139138
headers=headers,
140139
data=chunk,
@@ -180,7 +179,7 @@ def finish(self, file_handle: str) -> str:
180179
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
181180

182181
LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
183-
resp = requests.post(
182+
resp = request_post(
184183
url,
185184
headers=headers,
186185
json=data,

0 commit comments

Comments
 (0)