Set stdout and stderr to blocking after running an external command

Some nodejs executables set stdout and/or stderr to non-blocking, which
can trigger an IOError when the collectstatic command prints more than
1024 bytes at a time to stdout or stderr

Author: fdintino

pipeline/compilers/__init__.py

@@ -12,7 +12,7 @@
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompilerError
-from pipeline.utils import to_class
+from pipeline.utils import to_class, set_std_streams_blocking
 
 
 class Compiler(object):
@@ -120,6 +120,7 @@ def execute_command(self, command, cwd=None, stdout_captured=None):
                                              stdout=stdout,
                                              stderr=subprocess.PIPE)
                 _, stderr = compiling.communicate()
+                set_std_streams_blocking()
 
             if compiling.returncode != 0:
                 stdout_captured = None  # Don't save erroneous result.

pipeline/compressors/__init__.py

@@ -14,7 +14,7 @@
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompressorError
-from pipeline.utils import to_class, relpath
+from pipeline.utils import to_class, relpath, set_std_streams_blocking
 
 URL_DETECTOR = r"""url\((['"]){0,1}\s*(.*?)["']{0,1}\)"""
 URL_REPLACER = r"""url\(__EMBED__(.+?)(\?\d+)?\)"""
@@ -248,6 +248,7 @@ def execute_command(self, command, content):
         if content:
             content = smart_bytes(content)
         stdout, stderr = pipe.communicate(content)
+        set_std_streams_blocking()
         if stderr.strip() and pipe.returncode != 0:
             raise CompressorError(stderr)
         elif self.verbose:

pipeline/utils.py

@@ -1,8 +1,16 @@
 from __future__ import unicode_literals
 
+try:
+    import fcntl
+except ImportError:
+    # windows
+    fcntl = None
+
 import importlib
 import mimetypes
 import posixpath
+import os
+import sys
 
 try:
     from urllib.parse import quote
@@ -54,3 +62,12 @@ def relpath(path, start=posixpath.curdir):
     if not rel_list:
         return posixpath.curdir
     return posixpath.join(*rel_list)
+
+
+def set_std_streams_blocking():
+    if not fcntl:
+        return
+    for f in (sys.__stdout__, sys.__stderr__):
+        fileno = f.fileno()
+        flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
+        fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)