Merge pull request #1 from common-workflow-language/master

merge new changes

Author: kapilkd13

cwltool/cwlNodeEngine.js

@@ -1,7 +1,7 @@
 "use strict";
-process.stdin.setEncoding('utf8');
+process.stdin.setEncoding("utf8");
 var incoming = "";
-process.stdin.on('data', function(chunk) {
+process.stdin.on("data", function(chunk) {
   incoming += chunk;
   var i = incoming.indexOf("\n");
   if (i > -1) {
@@ -10,4 +10,4 @@ process.stdin.on('data', function(chunk) {
     process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {})) + "\n");
   }
 });
-process.stdin.on('end', process.exit);
+process.stdin.on("end", process.exit);

cwltool/job.py

@@ -135,15 +135,22 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
 
         img_id = None
         env = None  # type: Union[MutableMapping[Text, Text], MutableMapping[str, str]]
-        if docker_req and kwargs.get("use_container") is not False:
-            env = os.environ
-            img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)
-        elif kwargs.get("default_container", None) is not None:
-            env = os.environ
-            img_id = kwargs.get("default_container")
-
-        if docker_is_req and img_id is None:
-            raise WorkflowException("Docker is required for running this tool.")
+        try:
+            if docker_req and kwargs.get("use_container") is not False:
+                env = os.environ
+                img_id = docker.get_from_requirements(docker_req, True, pull_image)
+            elif kwargs.get("default_container", None) is not None:
+                env = os.environ
+                img_id = kwargs.get("default_container")
+
+            if docker_req and img_id is None and kwargs.get("use_container"):
+                raise Exception("Docker image not available")
+        except Exception as e:
+            _logger.debug("Docker error", exc_info=True)
+            if docker_is_req:
+                raise WorkflowException("Docker is required to run this tool: %s" % e)
+            else:
+                raise WorkflowException("Docker is not available for this tool, try --no-container to disable Docker: %s" % e)
 
         if img_id:
             runtime = ["docker", "run", "-i"]

cwltool/sandboxjs.py

@@ -33,9 +33,15 @@ def new_js_proc():
     trynodes = ("nodejs", "node")
     for n in trynodes:
         try:
-            nodejs = subprocess.Popen([n, "--eval", nodecode], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            if subprocess.check_output([n, "--eval", "process.stdout.write('t')"]) != "t":
+                continue
+            nodejs = subprocess.Popen([n, "--eval", nodecode],
+                                      stdin=subprocess.PIPE,
+                                      stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
             break
+        except subprocess.CalledProcessError:
+            pass
         except OSError as e:
             if e.errno == errno.ENOENT:
                 pass
@@ -103,26 +109,28 @@ def term():
     stdout_buf = BytesIO()
     stderr_buf = BytesIO()
 
-    completed = []  # type: List[BytesIO]
-    while len(completed) < 3:
-        rready, wready, _ = select.select([nodejs.stdout, nodejs.stderr], [nodejs.stdin], [])
-        if nodejs.stdin in wready:
-            b = stdin_buf.read(select.PIPE_BUF)
-            if b:
-                os.write(nodejs.stdin.fileno(), b)
-            elif stdin_buf not in completed:
-                completed.append(stdin_buf)
-        for pipes in ((nodejs.stdout, stdout_buf), (nodejs.stderr, stderr_buf)):
-            if pipes[0] in rready:
-                b = os.read(pipes[0].fileno(), select.PIPE_BUF)
+    rselect = [nodejs.stdout, nodejs.stderr]  # type: List[BytesIO]
+    wselect = [nodejs.stdin]  # type: List[BytesIO]
+    while (len(wselect) + len(rselect)) > 0:
+        rready, wready, _ = select.select(rselect, wselect, [])
+        try:
+            if nodejs.stdin in wready:
+                b = stdin_buf.read(select.PIPE_BUF)
                 if b:
-                    pipes[1].write(b)
-                elif pipes[1] not in completed:
-                    completed.append(pipes[1])
-        if stdout_buf.getvalue().endswith("\n"):
-            for buf in (stdout_buf, stderr_buf):
-                if buf not in completed:
-                    completed.append(buf)
+                    os.write(nodejs.stdin.fileno(), b)
+                else:
+                    wselect = []
+            for pipes in ((nodejs.stdout, stdout_buf), (nodejs.stderr, stderr_buf)):
+                if pipes[0] in rready:
+                    b = os.read(pipes[0].fileno(), select.PIPE_BUF)
+                    if b:
+                        pipes[1].write(b)
+                    else:
+                        rselect.remove(pipes[0])
+            if stdout_buf.getvalue().endswith("\n"):
+                rselect = []
+        except OSError as e:
+            break
     tm.cancel()
 
     stdin_buf.close()
@@ -143,6 +151,8 @@ def stdfmt(data):  # type: (unicode) -> unicode
             return "\n" + data.strip()
         return data
 
+    nodejs.poll()
+
     if debug:
         info = u"returncode was: %s\nscript was:\n%s\nstdout was: %s\nstderr was: %s\n" %\
                (nodejs.returncode, fn_linenum(), stdfmt(stdoutdata), stdfmt(stderrdata))

requirements.txt

@@ -3,5 +3,5 @@ ruamel.yaml==0.13.7
 rdflib==4.2.1
 rdflib-jsonld==0.4.0
 shellescape==3.4.1
-schema-salad>=2.1.20170208112505,<3
+schema-salad>=2.3.20170302225134,<3
 typing==3.5.2.2 ; python_version>="2.7"

setup.py

@@ -50,7 +50,7 @@
           'ruamel.yaml >= 0.12.4',
           'rdflib >= 4.2.2, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',
-          'schema-salad >= 2.2.20170222151604, < 3',
+          'schema-salad >= 2.3.20170302225134, < 3',
           'typing >= 3.5.2, < 3.6',
           'six >= 1.10.0',
 

tests/test_examples.py

@@ -6,6 +6,7 @@
 import cwltool.process
 import cwltool.workflow
 from .util import get_data
+from cwltool.main import main
 
 class TestParamMatching(unittest.TestCase):
     def test_params(self):
@@ -336,6 +337,11 @@ def test_lifting(self):
             echo = f.make(get_data("tests/test_bad_outputs_wf.cwl"))
             self.assertEqual(echo(inp="foo"), {"out": "foo\n"})
 
+class TestPrintDot(unittest.TestCase):
+    def test_print_dot(self):
+        # Require that --enable-ext is provided.
+        self.assertEquals(main(["--print-dot", get_data('tests/wf/revsort.cwl')]), 0)
+
 
 if __name__ == '__main__':
     unittest.main()

tox.ini

@@ -15,15 +15,15 @@ deps = -rrequirements.txt
 commands = make mypy
 whitelist_externals = make
 deps =
-  mypy>=0.470
+  mypy==0.470
   typed-ast==0.6.3
   -rrequirements.txt
 
 [testenv:py35-mypy]
 commands = make mypy3
 whitelist_externals = make
 deps =
-  mypy>=0.470
+  mypy==0.470
   typed-ast==0.6.3
   -rrequirements.txt