Add option to use ManifestStaticFilesStorage

Author: johanneswilm

README.md

@@ -61,7 +61,7 @@ You can continue to load other resources such as CSS files as before using the `
 
 
 NPM.JS dependencies
------------
+-------------------
 
 1. Add package.json files into one or more of your apps. All package.json files will be merged.
 
@@ -72,7 +72,7 @@ NPM.JS dependencies
 4. Run `./manage.py runserver`.
 
 Referring to the transpile version within JavaScript sources
-------
+------------------------------------------------------------
 
 In your JavaScript sources, you can refer to the version string of the last transpile run like this::
 
@@ -81,3 +81,20 @@ In your JavaScript sources, you can refer to the version string of the last tran
 For example::
 
         let downloadJS = `download.js?v=${transpile.VERSION}` // Latest version of transpiled version of download.mjs
+
+
+ManifestStaticFilesStorage
+--------------------------
+If you use `ManifestStaticFilesStorage`, import it from `npm_mjs.storage` like this:
+
+```py
+from npm_mjs.storage import ManifestStaticFilesStorage
+```
+
+If you use that version, you can refer to other static files within your JavaScript files using the `staticUrl()` function like this:
+
+```js
+const cssUrl = staticUrl('/css/document.css')
+```
+
+Note that you will need to use absolute paths starting from the `STATIC_ROOT` for the `staticUrl()` function. Different from the default `ManifestStaticFilesStorage`, our version will generally interprete file urls starting with a slash as being relative to the `STATIC_ROOT`.

npm_mjs/management/commands/collectstatic.py

@@ -1,9 +1,11 @@
 from django.contrib.staticfiles.management.commands.collectstatic import (
     Command as CollectStaticCommand,
 )
+from django.apps import apps
 
 
 class Command(CollectStaticCommand):
+
     def set_options(self, *args, **options):
         return_value = super(Command, self).set_options(*args, **options)
         self.ignore_patterns += [

npm_mjs/management/commands/makemessages.py

@@ -0,0 +1,80 @@
+import os
+import tempfile
+import shutil
+import urllib
+
+from django.core.management.commands import makemessages
+from django.core.management import call_command
+from django.core.management.utils import popen_wrapper
+
+from base.management import BaseCommand
+
+# This makes makemessages create both translations for Python and JavaScript
+# code in one go.
+#
+# Additionally, it works around xgettext not being able to work with ES2015
+# template strings [1] by transpiling the JavaScript files in a special way to
+# replace all template strings.
+#
+# [1] https://savannah.gnu.org/bugs/?50920
+
+
+class Command(makemessages.Command, BaseCommand):
+    def handle(self, *args, **options):
+        call_command("transpile")
+        options["ignore_patterns"] += [
+            "venv",
+            ".direnv",
+            "node_modules",
+            "static-transpile",
+        ]
+        options["domain"] = "django"
+        super().handle(*args, **options)
+        options["domain"] = "djangojs"
+        self.temp_dir_out = tempfile.mkdtemp()
+        self.temp_dir_in = tempfile.mkdtemp()
+        super().handle(*args, **options)
+        shutil.rmtree(self.temp_dir_in)
+        shutil.rmtree(self.temp_dir_out)
+
+    def process_locale_dir(self, locale_dir, files):
+        if self.domain == "djangojs":
+            for file in files:
+                # We need to copy the JS files first, as otherwise babel will
+                # attempt to read package.json files in subdirs, such as
+                # base/package.json
+                in_path = urllib.parse.urljoin(
+                    self.temp_dir_in + "/", file.dirpath
+                )
+                os.makedirs(in_path, exist_ok=True)
+                in_file = urllib.parse.urljoin(in_path + "/", file.file)
+                shutil.copy2(file.path, in_file)
+                out_path = urllib.parse.urljoin(
+                    self.temp_dir_out + "/", file.dirpath
+                )
+                file.dirpath = out_path
+            os.chdir(".transpile/")
+            out, err, status = popen_wrapper(
+                [
+                    "npm",
+                    "run",
+                    "babel-transform-template-literals",
+                    "--",
+                    "--out-dir",
+                    self.temp_dir_out,
+                    self.temp_dir_in,
+                ]
+            )
+            os.chdir("../")
+
+        super().process_locale_dir(locale_dir, files)
+
+    def write_po_file(self, potfile, locale):
+        if self.domain == "djangojs":
+            with open(potfile, encoding="utf-8") as fp:
+                msgs = fp.read()
+            # Remove temp dir path info
+            msgs = msgs.replace(self.temp_dir_out + "/", "")
+            with open(potfile, "w", encoding="utf-8") as fp:
+                fp.write(msgs)
+        super().write_po_file(potfile, locale)

npm_mjs/management/commands/transpile.py

@@ -7,7 +7,7 @@
 import time
 import json
 from npm_mjs.paths import PROJECT_PATH, TRANSPILE_CACHE_PATH, STATIC_ROOT
-from npm_mjs.tools import get_last_run, set_last_run
+from npm_mjs.tools import set_last_run
 
 from urllib.parse import urljoin
 from django.core.management.base import BaseCommand
@@ -237,7 +237,7 @@ def handle(self, *args, **options):
         )
         transpile = {
             "OUT_DIR": out_dir,
-            "VERSION": get_last_run("transpile"),
+            "VERSION": start,
             "BASE_URL": transpile_base_url,
             "ENTRIES": entries,
             "STATIC_FRONTEND_FILES": list(

npm_mjs/management/commands/webpack.config.template.js

@@ -14,8 +14,15 @@ const baseRule = {
     }
 }
 
+const predefinedVariables = {
+    "transpile_VERSION": transpile.VERSION
+}
+
 if (settings.DEBUG) {
     baseRule.exclude = /node_modules/
+    predefinedVariables.staticUrl = `(url => (${settings.STATIC_URL} + url).replace(/[\/]+/g, '/'))`
+} else if (settings.STATICFILES_STORAGE !== "npm_mjs.storage.ManifestStaticFilesStorage") {
+    predefinedVariables.staticUrl = `(url => (${settings.STATIC_URL} + url).replace(/[\/]+/g, '/') + "?v=" + transpile_VERSION)`
 }
 
 module.exports = { // eslint-disable-line no-undef
@@ -29,9 +36,7 @@ module.exports = { // eslint-disable-line no-undef
         publicPath: transpile.BASE_URL
     },
     plugins: [
-        new webpack.DefinePlugin({
-            "transpile_VERSION": transpile.VERSION
-        })
+        new webpack.DefinePlugin(predefinedVariables)
     ],
     entry: transpile.ENTRIES
 }

npm_mjs/package.json

@@ -5,12 +5,14 @@
   "description": "Install dependencies for ES6 transpilation",
   "private": true,
   "dependencies": {
-    "@babel/preset-env": "^7.12.1",
-    "@babel/core": "7.14.6",
+    "@babel/preset-env": "^7.19.1",
+    "@babel/core": "7.19.1",
+    "@babel/cli": "7.18.10",
     "@babel/plugin-syntax-dynamic-import": "7.8.3",
     "@babel/plugin-proposal-optional-chaining": "^7.12.1",
-    "babel-loader": "8.2.2",
-    "webpack": "5.44.0",
-    "webpack-cli": "4.7.2"
+    "@babel/plugin-transform-template-literals": "^7.18.6",
+    "babel-loader": "8.2.5",
+    "webpack": "5.74.0",
+    "webpack-cli": "4.10.0"
   }
 }

npm_mjs/storage.py

@@ -0,0 +1,86 @@
+import os
+import posixpath
+import re
+from urllib.parse import unquote, urldefrag, urljoin
+
+from django.contrib.staticfiles.storage import ManifestStaticFilesStorage as DefaultManifestStaticFilesStorage
+from django.contrib.staticfiles.storage import HashedFilesMixin
+from django.conf import settings
+
+def add_js_static_pattern(pattern):
+    if pattern[0] == "*.js":
+        templates = pattern[1] + ((
+            '(?P<matched>staticUrl\\([\'"]{0,1}\\s*(?P<url>.*?)["\']{0,1}\\))',
+            "'%(url)s'",
+        ))
+        pattern = (pattern[0], templates)
+    return pattern
+
+
+class ManifestStaticFilesStorage(DefaultManifestStaticFilesStorage):
+    patterns = tuple(map(add_js_static_pattern, HashedFilesMixin.patterns))
+
+    def url_converter(self, name, hashed_files, template=None):
+        """
+        Return the custom URL converter for the given file name.
+        """
+        # Modified from
+        # https://github.com/django/django/blob/main/django/contrib/staticfiles/storage.py
+        # to handle absolute URLS
+
+        if template is None:
+            template = self.default_template
+
+        def converter(matchobj):
+            """
+            Convert the matched URL to a normalized and hashed URL.
+            This requires figuring out which files the matched URL resolves
+            to and calling the url() method of the storage.
+            """
+            matches = matchobj.groupdict()
+            matched = matches["matched"]
+            url = matches["url"]
+
+            # Ignore absolute/protocol-relative and data-uri URLs.
+            if re.match(r"^[a-z]+:", url):
+                return matched
+
+            # Strip off the fragment so a path-like fragment won't interfere.
+            url_path, fragment = urldefrag(url)
+
+            # Ignore URLs without a path
+            if not url_path:
+                return matched
+
+            if url_path.startswith("/"):
+                # Absolute paths are assumed to have their root at STATIC_ROOT
+                target_name = url_path[1:]
+            else:
+                # We're using the posixpath module to mix paths and URLs conveniently.
+                source_name = name if os.sep == "/" else name.replace(os.sep, "/")
+                target_name = posixpath.join(posixpath.dirname(source_name), url_path)
+
+            # Determine the hashed name of the target file with the storage backend.
+            hashed_url = self._url(
+                self._stored_name,
+                unquote(target_name),
+                force=True,
+                hashed_files=hashed_files,
+            )
+
+            transformed_url = "/".join(
+                url_path.split("/")[:-1] + hashed_url.split("/")[-1:]
+            )
+
+            # Restore the fragment that was stripped off earlier.
+            if fragment:
+                transformed_url += ("?#" if "?#" in url else "#") + fragment
+
+            if url_path.startswith("/"):
+                transformed_url = urljoin(settings.STATIC_URL, transformed_url[1:])
+
+            # Return the hashed version to the file
+            matches["url"] = unquote(transformed_url)
+            return template % matches
+
+        return converter

npm_mjs/templatetags/transpile.py

@@ -4,6 +4,7 @@
 from django import template
 from django.templatetags.static import StaticNode, PrefixNode
 from django.apps import apps
+from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
 
 from npm_mjs.tools import get_last_run
 
@@ -16,8 +17,10 @@ def handle_simple(cls, path):
         path = re.sub(r"^js/(.*)\.mjs", r"js/\1.js", path)
         if apps.is_installed("django.contrib.staticfiles"):
             from django.contrib.staticfiles.storage import staticfiles_storage
-
-            return staticfiles_storage.url(path) + "?v=%s" % get_last_run("transpile")
+            if isinstance(staticfiles_storage, ManifestStaticFilesStorage):
+                return staticfiles_storage.url(path)
+            else:
+                return staticfiles_storage.url(path) + "?v=%s" % get_last_run("transpile")
         else:
             return urljoin(
                 PrefixNode.handle_simple("STATIC_URL"), quote(path)

npm_mjs/tools.py

@@ -15,6 +15,7 @@ def load_last_name():
 
 
 def get_last_run(name):
+    global _last_run
     if name not in _last_run:
         load_last_name()
         if name not in _last_run: