Add sourcemap support

Author: fdintino

pipeline/compressors/__init__.py

@@ -4,6 +4,7 @@
 import os
 import posixpath
 import re
+import warnings
 
 from itertools import takewhile
 
@@ -55,29 +56,46 @@ def css_compressor(self):
 
     def compress_js(self, paths, templates=None, **kwargs):
         """Concatenate and compress JS files"""
+        compressor = self.js_compressor
+
+        if settings.PIPELINE_OUTPUT_SOURCEMAPS:
+            if hasattr(compressor, 'compress_js_with_source_map'):
+                if templates:
+                    warnings.warn("Source maps are not supported with javascript templates")
+                else:
+                    return compressor(verbose=self.verbose).compress_js_with_source_map(paths)
+
         js = self.concatenate(paths)
         if templates:
             js = js + self.compile_templates(templates)
 
         if not settings.PIPELINE_DISABLE_WRAPPER:
             js = "(function() {\n%s\n}).call(this);" % js
 
-        compressor = self.js_compressor
         if compressor:
             js = getattr(compressor(verbose=self.verbose), 'compress_js')(js)
 
-        return js
+        return js, None
 
     def compress_css(self, paths, output_filename, variant=None, **kwargs):
         """Concatenate and compress CSS files"""
-        css = self.concatenate_and_rewrite(paths, output_filename, variant)
         compressor = self.css_compressor
+
+        if settings.PIPELINE_OUTPUT_SOURCEMAPS:
+            if hasattr(compressor, 'compress_css_with_source_map'):
+                if variant == "datauri":
+                    warnings.warn("Source maps are not supported with datauri variant")
+                else:
+                    return (compressor(verbose=self.verbose)
+                                .compress_css_with_source_map(paths, output_filename))
+
+        css = self.concatenate_and_rewrite(paths, output_filename, variant)
         if compressor:
             css = getattr(compressor(verbose=self.verbose), 'compress_css')(css)
         if not variant:
-            return css
+            return css, None
         elif variant == "datauri":
-            return self.with_data_uri(css)
+            return self.with_data_uri(css), None
         else:
             raise CompressorError("\"%s\" is not a valid variant" % variant)
 
@@ -233,10 +251,11 @@ def filter_js(self, js):
 
 
 class SubProcessCompressor(CompressorBase):
-    def execute_command(self, command, content):
+    def execute_command(self, command, content, shell=True):
         import subprocess
-        pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
-                                stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdin = subprocess.PIPE if content else None
+        pipe = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
+                                stdin=stdin, stderr=subprocess.PIPE)
         if content:
             content = smart_bytes(content)
         stdout, stderr = pipe.communicate(content)

pipeline/compressors/closure.py

@@ -1,10 +1,71 @@
 from __future__ import unicode_literals
 
+import os
+import re
+import shlex
+import tempfile
+
+from django.contrib.staticfiles.storage import staticfiles_storage
+from django.utils import six
+
 from pipeline.conf import settings
 from pipeline.compressors import SubProcessCompressor
 
 
+source_map_re = re.compile((
+    "(?:"
+      "/\\*"
+      "(?:\\s*\r?\n(?://)?)?"
+      "(?:%(inner)s)"
+      "\\s*"
+      "\\*/"
+      "|"
+      "//(?:%(inner)s)"
+    ")"
+    "\\s*$") % {'inner': r"""[#@] sourceMappingURL=([^\s'"]*)"""})
+
+
 class ClosureCompressor(SubProcessCompressor):
+
     def compress_js(self, js):
         command = '%s %s' % (settings.PIPELINE_CLOSURE_BINARY, settings.PIPELINE_CLOSURE_ARGUMENTS)
         return self.execute_command(command, js)
+
+    def compress_js_with_source_map(self, paths):
+        args = re.split(r'\s+', settings.PIPELINE_CLOSURE_BINARY)
+        if settings.PIPELINE_CLOSURE_ARGUMENTS:
+            if isinstance(settings.PIPELINE_CLOSURE_ARGUMENTS, six.string_types):
+                args += shlex.split(settings.PIPELINE_CLOSURE_ARGUMENTS)
+            else:
+                args += settings.PIPELINE_CLOSURE_ARGUMENTS
+        abs_paths = []
+        for path in paths:
+            abs_path = staticfiles_storage.path(path)
+            args += [
+                '--source_map_location_mapping',
+                "%s|%s" % (abs_path, staticfiles_storage.url(path))]
+            abs_paths.append(abs_path)
+            with open(abs_path) as f:
+                content = f.read()
+            matches = source_map_re.search(content)
+            if matches:
+                input_source_map = filter(None, matches.groups())[0]
+                input_source_map_file = os.path.join(os.path.dirname(abs_path), input_source_map)
+                args += [
+                    '--source_map_input',
+                    "%s|%s" %  (abs_path, input_source_map_file)]
+
+        temp_file = tempfile.NamedTemporaryFile()
+
+        args += ["--create_source_map", temp_file.name]
+        for path in abs_paths:
+            args += ["--js", path]
+
+        js = self.execute_command(args, None, shell=False)
+
+        with open(temp_file.name) as f:
+            source_map = f.read()
+
+        temp_file.close()
+
+        return js, source_map

pipeline/conf.py

@@ -25,6 +25,8 @@
 
     'PIPELINE_DISABLE_WRAPPER': False,
 
+    'PIPELINE_OUTPUT_SOURCEMAPS': False,
+
     'PIPELINE_CSSTIDY_BINARY': '/usr/bin/env csstidy',
     'PIPELINE_CSSTIDY_ARGUMENTS': '--template=highest',
 

pipeline/packager.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+import os.path
 
 from django.contrib.staticfiles.storage import staticfiles_storage
 from django.contrib.staticfiles.finders import find
@@ -92,24 +93,38 @@ def individual_url(self, filename):
 
     def pack_stylesheets(self, package, **kwargs):
         return self.pack(package, self.compressor.compress_css, css_compressed,
+                         compress_type='css',
                          output_filename=package.output_filename,
                          variant=package.variant, **kwargs)
 
     def compile(self, paths, force=False):
         return self.compiler.compile(paths, force=force)
 
-    def pack(self, package, compress, signal, **kwargs):
+    def pack(self, package, compress, signal, compress_type, **kwargs):
         output_filename = package.output_filename
         if self.verbose:
             print("Saving: %s" % output_filename)
         paths = self.compile(package.paths, force=True)
-        content = compress(paths, **kwargs)
+        content, source_map = compress(paths, **kwargs)
+        if source_map is not None:
+            source_map_output_filename = output_filename + '.map'
+            if self.verbose:
+                print("Saving: %s" % source_map_output_filename)
+            self.save_file(source_map_output_filename, source_map)
+            source_map_comment = "sourceMappingURL=%s" % (
+                os.path.basename(staticfiles_storage.url(source_map_output_filename)))
+            if compress_type == 'js':
+                content += "\n//# %s" % source_map_comment
+            else:
+                content += "\n/*# %s */" % source_map_comment
+            yield source_map_output_filename
         self.save_file(output_filename, content)
         signal.send(sender=self, package=package, **kwargs)
-        return output_filename
+        yield output_filename
 
     def pack_javascripts(self, package, **kwargs):
-        return self.pack(package, self.compressor.compress_js, js_compressed, templates=package.templates, **kwargs)
+        return self.pack(package, self.compressor.compress_js, js_compressed,
+                         compress_type='js', templates=package.templates, **kwargs)
 
     def pack_templates(self, package):
         return self.compressor.compile_templates(package.templates)

pipeline/storage.py

@@ -21,18 +21,22 @@ def post_process(self, paths, dry_run=False, **options):
         packager = Packager(storage=self)
         for package_name in packager.packages['css']:
             package = packager.package_for('css', package_name)
-            output_file = package.output_filename
             if self.packing:
-                packager.pack_stylesheets(package)
-            paths[output_file] = (self, output_file)
-            yield output_file, output_file, True
+                output_files = packager.pack_stylesheets(package)
+            else:
+                output_files = [package.output_filename]
+            for output_file in output_files:
+                paths[output_file] = (self, output_file)
+                yield output_file, output_file, True
         for package_name in packager.packages['js']:
             package = packager.package_for('js', package_name)
-            output_file = package.output_filename
             if self.packing:
-                packager.pack_javascripts(package)
-            paths[output_file] = (self, output_file)
-            yield output_file, output_file, True
+                output_files = packager.pack_javascripts(package)
+            else:
+                output_files = [package.output_filename]
+            for output_file in output_files:
+                paths[output_file] = (self, output_file)
+                yield output_file, output_file, True
 
         super_class = super(PipelineMixin, self)
         if hasattr(super_class, 'post_process'):