Skip to content

Commit 14c1bc7

Browse files
AA-TurnerAA-Turner
committed
Rearrange sphinx.ext.intersphinx._load 
1 parent 3439337 commit 14c1bc7

File tree

1 file changed

+167
-167
lines changed

1 file changed

+167
-167
lines changed

‎sphinx/ext/intersphinx/_load.py‎

Lines changed: 167 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -25,77 +25,120 @@
2525
from sphinx.util.typing import Inventory
2626

2727

28-
def _strip_basic_auth(url: str) -> str:
29-
"""Returns *url* with basic auth credentials removed. Also returns the
30-
basic auth username and password if they're present in *url*.
31-
32-
E.g.: https://user:[email protected] => https://example.com
33-
34-
*url* need not include basic auth credentials.
35-
36-
:param url: url which may or may not contain basic auth credentials
37-
:type url: ``str``
38-
39-
:return: *url* with any basic auth creds removed
40-
:rtype: ``str``
41-
"""
42-
frags = list(urlsplit(url))
43-
# swap out "user[:pass]@hostname" for "hostname"
44-
if '@' in frags[1]:
45-
frags[1] = frags[1].split('@')[1]
46-
return urlunsplit(frags)
47-
48-
49-
def _read_from_url(url: str, *, config: Config) -> IO:
50-
"""Reads data from *url* with an HTTP *GET*.
51-
52-
This function supports fetching from resources which use basic HTTP auth as
53-
laid out by RFC1738 § 3.1. See § 5 for grammar definitions for URLs.
54-
55-
.. seealso:
28+
def normalize_intersphinx_mapping(app: Sphinx, config: Config) -> None:
29+
for key, value in config.intersphinx_mapping.copy().items():
30+
try:
31+
if isinstance(value, (list, tuple)):
32+
# new format
33+
name, (uri, inv) = key, value
34+
if not isinstance(name, str):
35+
LOGGER.warning(__('intersphinx identifier %r is not string. Ignored'),
36+
name)
37+
config.intersphinx_mapping.pop(key)
38+
continue
39+
else:
40+
# old format, no name
41+
# xref RemovedInSphinx80Warning
42+
name, uri, inv = None, key, value
43+
msg = (
44+
"The pre-Sphinx 1.0 'intersphinx_mapping' format is "
45+
"deprecated and will be removed in Sphinx 8. Update to the "
46+
"current format as described in the documentation. "
47+
f"Hint: \"intersphinx_mapping = {{'<name>': {(uri, inv)!r}}}\"."
48+
"https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping" # NoQA: E501
49+
)
50+
LOGGER.warning(msg)
5651

57-
https://www.ietf.org/rfc/rfc1738.txt
52+
if not isinstance(inv, tuple):
53+
config.intersphinx_mapping[key] = (name, (uri, (inv,)))
54+
else:
55+
config.intersphinx_mapping[key] = (name, (uri, inv))
56+
except Exception as exc:
57+
LOGGER.warning(__('Failed to read intersphinx_mapping[%s], ignored: %r'), key, exc)
58+
config.intersphinx_mapping.pop(key)
5859

59-
:param url: URL of an HTTP resource
60-
:type url: ``str``
6160

62-
:return: data read from resource described by *url*
63-
:rtype: ``file``-like object
64-
"""
65-
r = requests.get(url, stream=True, timeout=config.intersphinx_timeout,
66-
_user_agent=config.user_agent,
67-
_tls_info=(config.tls_verify, config.tls_cacerts))
68-
r.raise_for_status()
69-
r.raw.url = r.url
70-
# decode content-body based on the header.
71-
# ref: https://github.com/psf/requests/issues/2155
72-
r.raw.read = functools.partial(r.raw.read, decode_content=True)
73-
return r.raw
61+
def load_mappings(app: Sphinx) -> None:
62+
"""Load all intersphinx mappings into the environment."""
63+
now = int(time.time())
64+
inventories = InventoryAdapter(app.builder.env)
65+
intersphinx_cache: dict[str, InventoryCacheEntry] = inventories.cache
7466

67+
with concurrent.futures.ThreadPoolExecutor() as pool:
68+
futures = []
69+
name: str | None
70+
uri: str
71+
invs: tuple[str | None, ...]
72+
for name, (uri, invs) in app.config.intersphinx_mapping.values():
73+
futures.append(pool.submit(
74+
fetch_inventory_group, name, uri, invs, intersphinx_cache, app, now,
75+
))
76+
updated = [f.result() for f in concurrent.futures.as_completed(futures)]
7577

76-
def _get_safe_url(url: str) -> str:
77-
"""Gets version of *url* with basic auth passwords obscured. This function
78-
returns results suitable for printing and logging.
78+
if any(updated):
79+
inventories.clear()
7980

80-
E.g.: https://user:[email protected] => https://[email protected]
81+
# Duplicate values in different inventories will shadow each
82+
# other; which one will override which can vary between builds
83+
# since they are specified using an unordered dict. To make
84+
# it more consistent, we sort the named inventories and then
85+
# add the unnamed inventories last. This means that the
86+
# unnamed inventories will shadow the named ones but the named
87+
# ones can still be accessed when the name is specified.
88+
named_vals = []
89+
unnamed_vals = []
90+
for name, _expiry, invdata in intersphinx_cache.values():
91+
if name:
92+
named_vals.append((name, invdata))
93+
else:
94+
unnamed_vals.append((name, invdata))
95+
for name, invdata in sorted(named_vals) + unnamed_vals:
96+
if name:
97+
inventories.named_inventory[name] = invdata
98+
for type, objects in invdata.items():
99+
inventories.main_inventory.setdefault(type, {}).update(objects)
81100

82-
:param url: a url
83-
:type url: ``str``
84101

85-
:return: *url* with password removed
86-
:rtype: ``str``
87-
"""
88-
parts = urlsplit(url)
89-
if parts.username is None:
90-
return url
91-
else:
92-
frags = list(parts)
93-
if parts.port:
94-
frags[1] = f'{parts.username}@{parts.hostname}:{parts.port}'
102+
def fetch_inventory_group(
103+
name: str | None,
104+
uri: str,
105+
invs: tuple[str | None, ...],
106+
cache: dict[str, InventoryCacheEntry],
107+
app: Sphinx,
108+
now: int,
109+
) -> bool:
110+
cache_time = now - app.config.intersphinx_cache_limit * 86400
111+
failures = []
112+
try:
113+
for inv in invs:
114+
if not inv:
115+
inv = posixpath.join(uri, INVENTORY_FILENAME)
116+
# decide whether the inventory must be read: always read local
117+
# files; remote ones only if the cache time is expired
118+
if '://' not in inv or uri not in cache or cache[uri][1] < cache_time:
119+
safe_inv_url = _get_safe_url(inv)
120+
LOGGER.info(__('loading intersphinx inventory from %s...'), safe_inv_url)
121+
try:
122+
invdata = fetch_inventory(app, uri, inv)
123+
except Exception as err:
124+
failures.append(err.args)
125+
continue
126+
if invdata:
127+
cache[uri] = name, now, invdata
128+
return True
129+
return False
130+
finally:
131+
if failures == []:
132+
pass
133+
elif len(failures) < len(invs):
134+
LOGGER.info(__("encountered some issues with some of the inventories,"
135+
" but they had working alternatives:"))
136+
for fail in failures:
137+
LOGGER.info(*fail)
95138
else:
96-
frags[1] = f'{parts.username}@{parts.hostname}'
97-
98-
return urlunsplit(frags)
139+
issues = '\n'.join(f[0] % f[1:] for f in failures)
140+
LOGGER.warning(__("failed to reach any of the inventories "
141+
"with the following issues:") + "\n" + issues)
99142

100143

101144
def fetch_inventory(app: Sphinx, uri: str, inv: str) -> Inventory:
@@ -135,117 +178,74 @@ def fetch_inventory(app: Sphinx, uri: str, inv: str) -> Inventory:
135178
return invdata
136179

137180

138-
def fetch_inventory_group(
139-
name: str | None,
140-
uri: str,
141-
invs: tuple[str | None, ...],
142-
cache: dict[str, InventoryCacheEntry],
143-
app: Sphinx,
144-
now: int,
145-
) -> bool:
146-
cache_time = now - app.config.intersphinx_cache_limit * 86400
147-
failures = []
148-
try:
149-
for inv in invs:
150-
if not inv:
151-
inv = posixpath.join(uri, INVENTORY_FILENAME)
152-
# decide whether the inventory must be read: always read local
153-
# files; remote ones only if the cache time is expired
154-
if '://' not in inv or uri not in cache or cache[uri][1] < cache_time:
155-
safe_inv_url = _get_safe_url(inv)
156-
LOGGER.info(__('loading intersphinx inventory from %s...'), safe_inv_url)
157-
try:
158-
invdata = fetch_inventory(app, uri, inv)
159-
except Exception as err:
160-
failures.append(err.args)
161-
continue
162-
if invdata:
163-
cache[uri] = name, now, invdata
164-
return True
165-
return False
166-
finally:
167-
if failures == []:
168-
pass
169-
elif len(failures) < len(invs):
170-
LOGGER.info(__("encountered some issues with some of the inventories,"
171-
" but they had working alternatives:"))
172-
for fail in failures:
173-
LOGGER.info(*fail)
181+
def _get_safe_url(url: str) -> str:
182+
"""Gets version of *url* with basic auth passwords obscured. This function
183+
returns results suitable for printing and logging.
184+
185+
E.g.: https://user:[email protected] => https://[email protected]
186+
187+
:param url: a url
188+
:type url: ``str``
189+
190+
:return: *url* with password removed
191+
:rtype: ``str``
192+
"""
193+
parts = urlsplit(url)
194+
if parts.username is None:
195+
return url
196+
else:
197+
frags = list(parts)
198+
if parts.port:
199+
frags[1] = f'{parts.username}@{parts.hostname}:{parts.port}'
174200
else:
175-
issues = '\n'.join(f[0] % f[1:] for f in failures)
176-
LOGGER.warning(__("failed to reach any of the inventories "
177-
"with the following issues:") + "\n" + issues)
201+
frags[1] = f'{parts.username}@{parts.hostname}'
178202

203+
return urlunsplit(frags)
179204

180-
def load_mappings(app: Sphinx) -> None:
181-
"""Load all intersphinx mappings into the environment."""
182-
now = int(time.time())
183-
inventories = InventoryAdapter(app.builder.env)
184-
intersphinx_cache: dict[str, InventoryCacheEntry] = inventories.cache
185205

186-
with concurrent.futures.ThreadPoolExecutor() as pool:
187-
futures = []
188-
name: str | None
189-
uri: str
190-
invs: tuple[str | None, ...]
191-
for name, (uri, invs) in app.config.intersphinx_mapping.values():
192-
futures.append(pool.submit(
193-
fetch_inventory_group, name, uri, invs, intersphinx_cache, app, now,
194-
))
195-
updated = [f.result() for f in concurrent.futures.as_completed(futures)]
206+
def _strip_basic_auth(url: str) -> str:
207+
"""Returns *url* with basic auth credentials removed. Also returns the
208+
basic auth username and password if they're present in *url*.
196209
197-
if any(updated):
198-
inventories.clear()
210+
E.g.: https://user:[email protected] => https://example.com
199211
200-
# Duplicate values in different inventories will shadow each
201-
# other; which one will override which can vary between builds
202-
# since they are specified using an unordered dict. To make
203-
# it more consistent, we sort the named inventories and then
204-
# add the unnamed inventories last. This means that the
205-
# unnamed inventories will shadow the named ones but the named
206-
# ones can still be accessed when the name is specified.
207-
named_vals = []
208-
unnamed_vals = []
209-
for name, _expiry, invdata in intersphinx_cache.values():
210-
if name:
211-
named_vals.append((name, invdata))
212-
else:
213-
unnamed_vals.append((name, invdata))
214-
for name, invdata in sorted(named_vals) + unnamed_vals:
215-
if name:
216-
inventories.named_inventory[name] = invdata
217-
for type, objects in invdata.items():
218-
inventories.main_inventory.setdefault(type, {}).update(objects)
212+
*url* need not include basic auth credentials.
219213
214+
:param url: url which may or may not contain basic auth credentials
215+
:type url: ``str``
220216
221-
def normalize_intersphinx_mapping(app: Sphinx, config: Config) -> None:
222-
for key, value in config.intersphinx_mapping.copy().items():
223-
try:
224-
if isinstance(value, (list, tuple)):
225-
# new format
226-
name, (uri, inv) = key, value
227-
if not isinstance(name, str):
228-
LOGGER.warning(__('intersphinx identifier %r is not string. Ignored'),
229-
name)
230-
config.intersphinx_mapping.pop(key)
231-
continue
232-
else:
233-
# old format, no name
234-
# xref RemovedInSphinx80Warning
235-
name, uri, inv = None, key, value
236-
msg = (
237-
"The pre-Sphinx 1.0 'intersphinx_mapping' format is "
238-
"deprecated and will be removed in Sphinx 8. Update to the "
239-
"current format as described in the documentation. "
240-
f"Hint: \"intersphinx_mapping = {{'<name>': {(uri, inv)!r}}}\"."
241-
"https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping" # NoQA: E501
242-
)
243-
LOGGER.warning(msg)
217+
:return: *url* with any basic auth creds removed
218+
:rtype: ``str``
219+
"""
220+
frags = list(urlsplit(url))
221+
# swap out "user[:pass]@hostname" for "hostname"
222+
if '@' in frags[1]:
223+
frags[1] = frags[1].split('@')[1]
224+
return urlunsplit(frags)
244225

245-
if not isinstance(inv, tuple):
246-
config.intersphinx_mapping[key] = (name, (uri, (inv,)))
247-
else:
248-
config.intersphinx_mapping[key] = (name, (uri, inv))
249-
except Exception as exc:
250-
LOGGER.warning(__('Failed to read intersphinx_mapping[%s], ignored: %r'), key, exc)
251-
config.intersphinx_mapping.pop(key)
226+
227+
def _read_from_url(url: str, *, config: Config) -> IO:
228+
"""Reads data from *url* with an HTTP *GET*.
229+
230+
This function supports fetching from resources which use basic HTTP auth as
231+
laid out by RFC1738 § 3.1. See § 5 for grammar definitions for URLs.
232+
233+
.. seealso:
234+
235+
https://www.ietf.org/rfc/rfc1738.txt
236+
237+
:param url: URL of an HTTP resource
238+
:type url: ``str``
239+
240+
:return: data read from resource described by *url*
241+
:rtype: ``file``-like object
242+
"""
243+
r = requests.get(url, stream=True, timeout=config.intersphinx_timeout,
244+
_user_agent=config.user_agent,
245+
_tls_info=(config.tls_verify, config.tls_cacerts))
246+
r.raise_for_status()
247+
r.raw.url = r.url
248+
# decode content-body based on the header.
249+
# ref: https://github.com/psf/requests/issues/2155
250+
r.raw.read = functools.partial(r.raw.read, decode_content=True)
251+
return r.raw

0 commit comments

Comments
 (0)