Merge pull request #1950 from FFY00/linkcheck-local-remote

Restore linkcheck for specs

Author: webknjaz

.github/CODEOWNERS

@@ -1,6 +1,9 @@
 source/guides/github-actions-ci-cd-sample/* @webknjaz
 source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst @webknjaz
 
+# Sphinx extension
+pug_sphinx_extensions/ @FFY00
+
 # build-details.json
 source/specifications/build-details/ @FFY00
 source/specifications/specs/build-details-*.json @FFY00

pug_sphinx_extensions/__init__.py

@@ -0,0 +1,86 @@
+import os
+import pathlib
+import urllib
+
+import sphinx.application
+import sphinx.util.logging
+
+
+DOMAIN = "packaging.python.org"
+
+
+logger = sphinx.util.logging.getLogger(__name__)
+
+
+def resolve_local_html_link(app: sphinx.application.Sphinx, url_path: str) -> str:
+    """Takes path of a link pointing an HTML render of the current project,
+    and returns local path of the referenced document.
+
+    Support links to renders from both the `html` and `dirhtml` builders.
+
+    Example:
+
+    .. code-block:: python
+
+        >>> resolve_local_html_link('https://packaging.python.org/en/latest/flow/')
+        '{srcdir}/flow.rst'
+        >>> resolve_local_html_link('https://packaging.python.org/en/latest/flow.html')
+        '{srcdir}/flow.rst'
+        >>> resolve_local_html_link('https://packaging.python.org/en/latest/specifications/schemas/')
+        '{srcdir}/specifications/schemas/index.rst'
+        >>> resolve_local_html_link('https://packaging.python.org/en/latest/specifications/schemas/build-details-v1.0.schema.json')
+        '{html_extra_path0}/specifications/schemas/build-details-v1.0.schema.json'
+
+    """
+    # Search for document in html_extra_path
+    for entry in app.config.html_extra_path:
+        candidate = (app.confdir / entry / url_path).resolve()
+        if candidate.is_dir():
+            candidate = candidate / "index.html"
+        if candidate.exists():
+            return os.fspath(candidate)
+    # Convert html path to source path
+    url_path = url_path.removesuffix("/")  # Normalize
+    if url_path.endswith(".html"):
+        document = url_path.removesuffix(".html")
+    elif (candidate := f"{url_path}/index") in app.project.docnames:
+        document = candidate
+    else:
+        document = url_path
+    return app.env.doc2path(document)
+
+
+def rewrite_local_uri(app: sphinx.application.Sphinx, uri: str) -> str:
+    """Replace remote URIs targeting https://packaging.python.org/en/latest/...
+    with local ones, so that local changes are taken into account by linkcheck.
+
+    Additionally, resolve local relative links to html_extra_path.
+    """
+    local_uri = uri
+    parsed = urllib.parse.urlparse(uri)
+    # Links to https://packaging.python.org/en/latest/...
+    if parsed.hostname == DOMAIN and parsed.path.startswith("/en/latest/"):
+        document = parsed.path.removeprefix("/en/latest/")
+        local_uri = resolve_local_html_link(app, document)
+        logger.verbose(
+            f"{uri!s} is a remote URL that points to local sources, "
+            "replacing it with a local URL in linkcheck to take new changes "
+            "into account (pass -vv for more info)"
+        )
+        logger.debug(f"Replacing linkcheck URL {uri!r} with {local_uri!r}")
+    # Local relative links
+    if not parsed.scheme and not parsed.netloc and parsed.path:
+        full_path = pathlib.Path(app.env.docname).parent / parsed.path
+        local_uri = resolve_local_html_link(app, os.fspath(full_path))
+        if local_uri != uri:
+            logger.verbose(f"Local linkcheck URL {uri!r} resolved as {local_uri!r}")
+    return local_uri
+
+
+def setup(app: sphinx.application.Sphinx) -> dict[str, bool]:
+    app.connect("linkcheck-process-uri", rewrite_local_uri)
+
+    return {
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

source/conf.py

@@ -2,6 +2,11 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 import os
+import pathlib
+import sys
+
+_ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path.append(os.fspath(_ROOT))
 
 # Some options are only enabled for the main packaging.python.org deployment builds
 RTD_BUILD = bool(os.getenv("READTHEDOCS"))
@@ -22,6 +27,7 @@
 root_doc = "index"
 
 extensions = [
+    "pug_sphinx_extensions",
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
     "sphinx.ext.todo",
@@ -133,7 +139,6 @@
 
 linkcheck_ignore = [
     r"http://localhost:\d+",
-    r"https://packaging\.python\.org/en/latest/specifications/schemas/.*",
     r"https://test\.pypi\.org/project/example-package-YOUR-USERNAME-HERE",
     r"https://pypi\.org/manage/.*",
     r"https://test\.pypi\.org/manage/.*",
@@ -162,9 +167,6 @@
     # https://github.com/pypa/packaging.python.org/issues/1744
     r"https://pypi\.org/",
 ]
-linkcheck_exclude_documents = [
-    "specifications/schemas/index",
-]
 
 # -- Options for extlinks ----------------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html#configuration