html builder: Append CRC32 checksum to asset URIs (#11415)

Author: AA-Turner

CHANGES

@@ -22,6 +22,9 @@ Deprecated
 Features added
 --------------
 
+* #11415: Add a checksum to JavaScript and CSS asset URIs included within
+  generated HTML, using the CRC32 algorithm.
+
 Bugs fixed
 ----------
 

sphinx/builders/__init__.py

@@ -560,6 +560,9 @@ def write(
         with progress_message(__('preparing documents')):
             self.prepare_writing(docnames)
 
+        with progress_message(__('copying assets')):
+            self.copy_assets()
+
         if self.parallel_ok:
             # number of subprocesses is parallel-1 because the main process
             # is busy loading doctrees and doing write_doc_serialized()
@@ -620,6 +623,10 @@ def prepare_writing(self, docnames: set[str]) -> None:
         """A place where you can add logic before :meth:`write_doc` is run"""
         raise NotImplementedError
 
+    def copy_assets(self) -> None:
+        """Where assets (images, static files, etc) are copied before writing"""
+        pass
+
     def write_doc(self, docname: str, doctree: nodes.document) -> None:
         """Where you actually write something to the filesystem."""
         raise NotImplementedError

sphinx/builders/html/__init__.py

@@ -8,6 +8,7 @@
 import re
 import sys
 import warnings
+import zlib
 from datetime import datetime
 from os import path
 from typing import IO, Any, Iterable, Iterator, List, Tuple, Type
@@ -649,6 +650,12 @@ def get_doc_context(self, docname: str, body: str, metatags: str) -> dict[str, A
             'page_source_suffix': source_suffix,
         }
 
+    def copy_assets(self) -> None:
+        self.finish_tasks.add_task(self.copy_download_files)
+        self.finish_tasks.add_task(self.copy_static_files)
+        self.finish_tasks.add_task(self.copy_extra_files)
+        self.finish_tasks.join()
+
     def write_doc(self, docname: str, doctree: nodes.document) -> None:
         destination = StringOutput(encoding='utf-8')
         doctree.settings = self.docsettings
@@ -678,9 +685,6 @@ def finish(self) -> None:
         self.finish_tasks.add_task(self.gen_pages_from_extensions)
         self.finish_tasks.add_task(self.gen_additional_pages)
         self.finish_tasks.add_task(self.copy_image_files)
-        self.finish_tasks.add_task(self.copy_download_files)
-        self.finish_tasks.add_task(self.copy_static_files)
-        self.finish_tasks.add_task(self.copy_extra_files)
         self.finish_tasks.add_task(self.write_buildinfo)
 
         # dump the search index
@@ -1193,8 +1197,11 @@ def css_tag(css: Stylesheet) -> str:
             value = css.attributes[key]
             if value is not None:
                 attrs.append(f'{key}="{html.escape(value, True)}"')
-        attrs.append('href="%s"' % pathto(css.filename, resource=True))
-        return '<link %s />' % ' '.join(attrs)
+        uri = pathto(css.filename, resource=True)
+        if checksum := _file_checksum(app.outdir, css.filename):
+            uri += f'?v={checksum}'
+        attrs.append(f'href="{uri}"')
+        return f'<link {" ".join(attrs)} />'
 
     context['css_tag'] = css_tag
 
@@ -1217,14 +1224,17 @@ def js_tag(js: JavaScript) -> str:
                     if key == 'body':
                         body = value
                     elif key == 'data_url_root':
-                        attrs.append('data-url_root="%s"' % pathto('', resource=True))
+                        attrs.append(f'data-url_root="{pathto("", resource=True)}"')
                     else:
                         attrs.append(f'{key}="{html.escape(value, True)}"')
             if js.filename:
-                attrs.append('src="%s"' % pathto(js.filename, resource=True))
+                uri = pathto(js.filename, resource=True)
+                if checksum := _file_checksum(app.outdir, js.filename):
+                    uri += f'?v={checksum}'
+                attrs.append(f'src="{uri}"')
         else:
             # str value (old styled)
-            attrs.append('src="%s"' % pathto(js, resource=True))
+            attrs.append(f'src="{pathto(js, resource=True)}"')
 
         if attrs:
             return f'<script {" ".join(attrs)}>{body}</script>'
@@ -1234,6 +1244,21 @@ def js_tag(js: JavaScript) -> str:
     context['js_tag'] = js_tag
 
 
+def _file_checksum(outdir: str, filename: str) -> str:
+    # Don't generate checksums for HTTP URIs
+    if '://' in filename:
+        return ''
+    try:
+        # Ensure universal newline mode is used to avoid checksum differences
+        with open(path.join(outdir, filename), encoding='utf-8') as f:
+            content = f.read().encode(encoding='utf-8')
+    except FileNotFoundError:
+        return ''
+    if not content:
+        return ''
+    return f'{zlib.crc32(content):08x}'
+
+
 def setup_resource_paths(app: Sphinx, pagename: str, templatename: str,
                          context: dict, doctree: Node) -> None:
     """Set up relative resource paths."""

sphinx/builders/latex/__init__.py

@@ -254,6 +254,12 @@ def write_stylesheet(self) -> None:
             f.write('% Its contents depend on pygments_style configuration variable.\n\n')
             f.write(highlighter.get_stylesheet())
 
+    def copy_assets(self) -> None:
+        self.copy_support_files()
+
+        if self.config.latex_additional_files:
+            self.copy_latex_additional_files()
+
     def write(self, *ignored: Any) -> None:
         docwriter = LaTeXWriter(self)
         with warnings.catch_warnings():
@@ -267,6 +273,7 @@ def write(self, *ignored: Any) -> None:
 
         self.init_document_data()
         self.write_stylesheet()
+        self.copy_assets()
 
         for entry in self.document_data:
             docname, targetname, title, author, themename = entry[:5]
@@ -371,10 +378,6 @@ def assemble_doctree(
     def finish(self) -> None:
         self.copy_image_files()
         self.write_message_catalog()
-        self.copy_support_files()
-
-        if self.config.latex_additional_files:
-            self.copy_latex_additional_files()
 
     @progress_message(__('copying TeX support files'))
     def copy_support_files(self) -> None:

sphinx/builders/texinfo.py

@@ -85,6 +85,7 @@ def init_document_data(self) -> None:
 
     def write(self, *ignored: Any) -> None:
         self.init_document_data()
+        self.copy_assets()
         for entry in self.document_data:
             docname, targetname, title, author = entry[:4]
             targetname += '.texi'
@@ -168,7 +169,7 @@ def assemble_doctree(
             pendingnode.replace_self(newnodes)
         return largetree
 
-    def finish(self) -> None:
+    def copy_assets(self) -> None:
         self.copy_support_files()
 
     def copy_image_files(self, targetname: str) -> None:

sphinx/ext/graphviz.py

@@ -8,7 +8,7 @@
 import subprocess
 from os import path
 from subprocess import CalledProcessError
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from docutils import nodes
 from docutils.nodes import Node
@@ -20,7 +20,6 @@
 from sphinx.locale import _, __
 from sphinx.util import logging, sha1
 from sphinx.util.docutils import SphinxDirective, SphinxTranslator
-from sphinx.util.fileutil import copy_asset
 from sphinx.util.i18n import search_image_for_language
 from sphinx.util.nodes import set_source_info
 from sphinx.util.osutil import ensuredir
@@ -31,6 +30,9 @@
 from sphinx.writers.texinfo import TexinfoTranslator
 from sphinx.writers.text import TextTranslator
 
+if TYPE_CHECKING:
+    from sphinx.config import Config
+
 logger = logging.getLogger(__name__)
 
 
@@ -391,11 +393,9 @@ def man_visit_graphviz(self: ManualPageTranslator, node: graphviz) -> None:
     raise nodes.SkipNode
 
 
-def on_build_finished(app: Sphinx, exc: Exception) -> None:
-    if exc is None and app.builder.format == 'html':
-        src = path.join(sphinx.package_dir, 'templates', 'graphviz', 'graphviz.css')
-        dst = path.join(app.outdir, '_static')
-        copy_asset(src, dst)
+def on_config_inited(_app: Sphinx, config: Config) -> None:
+    css_path = path.join(sphinx.package_dir, 'templates', 'graphviz', 'graphviz.css')
+    config.html_static_path.append(css_path)
 
 
 def setup(app: Sphinx) -> dict[str, Any]:
@@ -412,5 +412,5 @@ def setup(app: Sphinx) -> dict[str, Any]:
     app.add_config_value('graphviz_dot_args', [], 'html')
     app.add_config_value('graphviz_output_format', 'png', 'html')
     app.add_css_file('graphviz.css')
-    app.connect('build-finished', on_build_finished)
+    app.connect('config-inited', on_config_inited)
     return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

tests/test_build_html.py

@@ -1186,19 +1186,32 @@ def test_assets_order(app):
     content = (app.outdir / 'index.html').read_text(encoding='utf8')
 
     # css_files
-    expected = ['_static/early.css', '_static/pygments.css', '_static/alabaster.css',
-                'https://example.com/custom.css', '_static/normal.css', '_static/late.css',
-                '_static/css/style.css', '_static/lazy.css']
-    pattern = '.*'.join('href="%s"' % f for f in expected)
-    assert re.search(pattern, content, re.S)
+    expected = [
+        '_static/early.css',
+        '_static/pygments.css?v=b3523f8e',
+        '_static/alabaster.css?v=039e1c02',
+        'https://example.com/custom.css',
+        '_static/normal.css',
+        '_static/late.css',
+        '_static/css/style.css',
+        '_static/lazy.css',
+    ]
+    pattern = '.*'.join(f'href="{re.escape(f)}"' for f in expected)
+    assert re.search(pattern, content, re.DOTALL), content
 
     # js_files
-    expected = ['_static/early.js',
-                '_static/doctools.js', '_static/sphinx_highlight.js',
-                'https://example.com/script.js', '_static/normal.js',
-                '_static/late.js', '_static/js/custom.js', '_static/lazy.js']
-    pattern = '.*'.join('src="%s"' % f for f in expected)
-    assert re.search(pattern, content, re.S)
+    expected = [
+        '_static/early.js',
+        '_static/doctools.js?v=888ff710',
+        '_static/sphinx_highlight.js?v=4825356b',
+        'https://example.com/script.js',
+        '_static/normal.js',
+        '_static/late.js',
+        '_static/js/custom.js',
+        '_static/lazy.js',
+    ]
+    pattern = '.*'.join(f'src="{re.escape(f)}"' for f in expected)
+    assert re.search(pattern, content, re.DOTALL), content
 
 
 @pytest.mark.sphinx('html', testroot='html_assets')

tests/test_theming.py

@@ -99,10 +99,10 @@ def test_dark_style(app, status, warning):
     assert (app.outdir / '_static' / 'pygments_dark.css').exists()
 
     result = (app.outdir / 'index.html').read_text(encoding='utf8')
-    assert '<link rel="stylesheet" type="text/css" href="_static/pygments.css" />' in result
+    assert '<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />' in result
     assert ('<link id="pygments_dark_css" media="(prefers-color-scheme: dark)" '
             'rel="stylesheet" type="text/css" '
-            'href="_static/pygments_dark.css" />') in result
+            'href="_static/pygments_dark.css?v=e15ddae3" />') in result
 
 
 @pytest.mark.sphinx(testroot='theming')