Add a view for collecting static files before serving them.

In a Django development setup, it's common to have a /static/ URL set up
to serve static media to the browser upon request. That's usually set to
either serve out of a specific directory or out of any configured
finders. However, the former doesn't guarantee any referenced files were
collected first, and the latter only considers source files (for the
purposes of timestamp calculation/caching) and not the
collected/compiled files.

This change introduces a new view, meant for use with Django's static
serving view, that will collect the referenced file before serving it,
guaranteeing that media like images and fonts are present.

To ensure that this view will return content quickly (especially
considering it will serve up any files referenced by the template tags),
an optimization was made to Collector to allow the caller to collect
only the given list of files. The view makes use of this to handle the
collection process fast.

Author: chipx86

pipeline/collector.py

@@ -6,6 +6,7 @@
 
 from django.contrib.staticfiles import finders
 from django.contrib.staticfiles.storage import staticfiles_storage
+from django.utils import six
 
 from pipeline.finders import PipelineFinder
 
@@ -26,7 +27,7 @@ def clear(self, path=""):
         for d in dirs:
             self.clear(os.path.join(path, d))
 
-    def collect(self, request=None):
+    def collect(self, request=None, files=[]):
         if self.request and self.request is request:
             return
         self.request = request
@@ -41,10 +42,17 @@ def collect(self, request=None):
                     prefixed_path = os.path.join(storage.prefix, path)
                 else:
                     prefixed_path = path
-                if prefixed_path not in found_files:
+
+                if (prefixed_path not in found_files and
+                    (not files or prefixed_path in files)):
                     found_files[prefixed_path] = (storage, path)
                     self.copy_file(path, prefixed_path, storage)
 
+                if files and len(files) == len(found_files):
+                    break
+
+        return six.iterkeys(found_files)
+
     def copy_file(self, path, prefixed_path, source_storage):
         # Delete the target file if needed or break
         if not self.delete_file(path, prefixed_path, source_storage):

pipeline/views.py

@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+from django.conf import settings as django_settings
+from django.core.exceptions import ImproperlyConfigured
+from django.views.static import serve
+
+from .collector import default_collector
+from .conf import settings
+
+
+def serve_static(request, path, insecure=False, **kwargs):
+    """Collect and serve static files.
+
+    This view serves up static files, much like Django's
+    :py:func:`~django.views.static.serve` view, with the addition that it
+    collects static files first (if enabled). This allows images, fonts, and
+    other assets to be served up without first loading a page using the
+    ``{% javascript %}`` or ``{% stylesheet %}`` template tags.
+
+    You can use this view by adding the following to any :file:`urls.py`::
+
+        urlpatterns += static('static/', view='pipeline.views.serve_static')
+    """
+    # Follow the same logic Django uses for determining access to the
+    # static-serving view.
+    if not django_settings.DEBUG and not insecure:
+        raise ImproperlyConfigured("The staticfiles view can only be used in "
+                                   "debug mode or if the --insecure "
+                                   "option of 'runserver' is used")
+
+    if not settings.PIPELINE_ENABLED and settings.PIPELINE_COLLECTOR_ENABLED:
+        # Collect only the requested file, in order to serve the result as
+        # fast as possible. This won't interfere with the template tags in any
+        # way, as those will still cause Django to collect all media.
+        default_collector.collect(request, files=[path])
+
+    return serve(request, path, document_root=django_settings.STATIC_ROOT,
+                 **kwargs)

tests/tests/__init__.py

@@ -7,6 +7,7 @@
     os.environ.setdefault('NUMBER_OF_PROCESSORS', '1')
 
 
+from .test_collector import *
 from .test_compiler import *
 from .test_compressor import *
 from .test_template import *
@@ -15,3 +16,4 @@
 from .test_packager import *
 from .test_storage import *
 from .test_utils import *
+from .test_views import *

tests/tests/test_collector.py

@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+
+import os
+
+from django.contrib.staticfiles import finders
+from django.test import TestCase
+
+from pipeline.collector import default_collector
+from pipeline.finders import PipelineFinder
+
+
+class CollectorTest(TestCase):
+    def tearDown(self):
+        super(CollectorTest, self).tearDown()
+
+        default_collector.clear()
+
+    def test_collect(self):
+        self.assertEqual(
+            set(default_collector.collect()),
+            set(self._get_collectable_files()))
+
+    def test_collect_with_files(self):
+        self.assertEqual(
+            set(default_collector.collect(files=[
+                'pipeline/js/first.js',
+                'pipeline/js/second.js',
+            ])),
+            set([
+                'pipeline/js/first.js',
+                'pipeline/js/second.js',
+            ]))
+
+    def _get_collectable_files(self):
+        for finder in finders.get_finders():
+            if not isinstance(finder, PipelineFinder):
+                for path, storage in finder.list(['CVS', '.*', '*~']):
+                    if getattr(storage, 'prefix', None):
+                        yield os.path.join(storage.prefix, path)
+                    else:
+                        yield path

tests/tests/test_views.py

@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+from django.contrib.staticfiles.storage import staticfiles_storage
+from django.core.exceptions import ImproperlyConfigured
+from django.http import Http404
+from django.test import RequestFactory, TestCase
+from django.test.utils import override_settings
+
+from pipeline.collector import default_collector
+from pipeline.views import serve_static
+from tests.utils import pipeline_settings
+
+
+@override_settings(DEBUG=True)
+@pipeline_settings(PIPELINE_COLLECTOR_ENABLED=True, PIPELINE_ENABLED=False)
+class ServeStaticViewsTest(TestCase):
+    def setUp(self):
+        super(ServeStaticViewsTest, self).setUp()
+
+        self.filename = 'pipeline/js/first.js'
+        self.storage = staticfiles_storage
+        self.request = RequestFactory().get('/static/%s' % self.filename)
+
+        default_collector.clear()
+
+    def tearDown(self):
+        super(ServeStaticViewsTest, self).tearDown()
+
+        default_collector.clear()
+        staticfiles_storage._setup()
+
+    def test_found(self):
+        self._test_found()
+
+    def test_not_found(self):
+        self._test_not_found('missing-file')
+
+    @override_settings(DEBUG=False)
+    def test_debug_false(self):
+        with self.assertRaises(ImproperlyConfigured):
+            serve_static(self.request, self.filename)
+
+        self.assertFalse(self.storage.exists(self.filename))
+
+    @override_settings(DEBUG=False)
+    def test_debug_false_and_insecure(self):
+        self._test_found(insecure=True)
+
+    @pipeline_settings(PIPELINE_ENABLED=True)
+    def test_pipeline_enabled_and_found(self):
+        self._write_content()
+        self._test_found()
+
+    @pipeline_settings(PIPELINE_ENABLED=True)
+    def test_pipeline_enabled_and_not_found(self):
+        self._test_not_found(self.filename)
+
+    @pipeline_settings(PIPELINE_COLLECTOR_ENABLED=False)
+    def test_collector_disabled_and_found(self):
+        self._write_content()
+        self._test_found()
+
+    @pipeline_settings(PIPELINE_COLLECTOR_ENABLED=False)
+    def test_collector_disabled_and_not_found(self):
+        self._test_not_found(self.filename)
+
+    def _write_content(self, content='abc123'):
+        """Write sample content to the test static file."""
+        with self.storage.open(self.filename, 'w') as f:
+            f.write(content)
+
+    def _test_found(self, **kwargs):
+        """Test that a file can be found and contains the correct content."""
+        response = serve_static(self.request, self.filename, **kwargs)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(self.storage.exists(self.filename))
+
+        if hasattr(response, 'streaming_content'):
+            content = b''.join(response.streaming_content)
+        else:
+            content = response.content
+
+        with self.storage.open(self.filename) as f:
+            self.assertEqual(f.read(), content)
+
+    def _test_not_found(self, filename):
+        """Test that a file could not be found."""
+        self.assertFalse(self.storage.exists(filename))
+
+        with self.assertRaises(Http404):
+            serve_static(self.request, filename)
+
+        self.assertFalse(self.storage.exists(filename))