Merge pull request #750 from common-workflow-language/dynresreq_sizes

Fill in sizes for files inside directories only if needed

Author: mr-c

appveyor.yml

@@ -25,13 +25,15 @@ environment:
       PYTHON_ARCH: "64"
 
 install:
-  - "%PYTHON%\\python.exe -m pip install -U wheel pytest pytest-xdist mock"
+  - "%PYTHON%\\python.exe -m pip install -U wheel pytest mock"
+    #- "%PYTHON%\\python.exe -m pip install -U wheel pytest pytest-xdist mock"
 
 build_script:
   - "%PYTHON%\\python.exe -m pip install ."
 
 test_script:
-  - "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider --junit-xml=tests.xml -n2"
+  - "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider --junit-xml=tests.xml"
+    #- "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider --junit-xml=tests.xml -n2"
 
 on_finish:
   - ps: |

cwltool/expression.py

@@ -222,6 +222,9 @@ def interpolate(scan, rootvars,
     parts.append(scan)
     return ''.join(parts)
 
+def needs_parsing(snippet):  # type: (Any) -> bool
+    return isinstance(snippet, (str, Text)) \
+        and ("$(" in snippet or "${" in snippet)
 
 def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
             context=None, pull_image=True, timeout=None, force_docker_pull=False,
@@ -237,7 +240,8 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
         u"self": context,
         u"runtime": runtime}
 
-    if isinstance(ex, (str, Text)) and ("$(" in ex or "${" in ex):
+    if needs_parsing(ex):
+        assert isinstance(ex, (str, Text))
         fullJS = False
         jslib = u""
         for r in reversed(requirements):

cwltool/main.py

@@ -42,7 +42,7 @@
                                     get_container_from_software_requirements)
 from .stdfsaccess import StdFsAccess
 from .update import ALLUPDATES, UPDATES
-from .utils import onWindows, windows_default_container_id
+from .utils import onWindows, windows_default_container_id, add_sizes
 
 _logger = logging.getLogger("cwltool")
 
@@ -227,17 +227,7 @@ def pathToLoc(p):
             p["location"] = p["path"]
             del p["path"]
 
-    def addSizes(p):
-        if 'location' in p:
-            try:
-                p["size"] = os.stat(p["location"][7:]).st_size  # strip off file://
-            except OSError:
-                pass
-        elif 'contents' in p:
-                p["size"] = len(p['contents'])
-        else:
-            return  # best effort
-
+  
     ns = {}  # type: Dict[Text, Union[Dict[Any, Any], Text, Iterable[Text]]]
     ns.update(t.metadata.get("$namespaces", {}))
     ld = Loader(ns)
@@ -247,7 +237,7 @@ def expand_formats(p):
             p["format"] = ld.expand_url(p["format"], "")
 
     visit_class(job_order_object, ("File", "Directory"), pathToLoc)
-    visit_class(job_order_object, ("File",), addSizes)
+    visit_class(job_order_object, ("File",), add_sizes)
     visit_class(job_order_object, ("File",), expand_formats)
     adjustDirObjs(job_order_object, trim_listing)
     normalizeFilesDirs(job_order_object)

cwltool/process.py

@@ -31,6 +31,7 @@
 from six.moves import urllib
 from six import iteritems, itervalues, string_types
 
+from . import expression
 from .validate_js import validate_js_expressions
 from .utils import cmp_like_py2
 from .builder import Builder
@@ -40,7 +41,8 @@
                          ensure_writable)
 from .secrets import SecretStore
 from .stdfsaccess import StdFsAccess
-from .utils import aslist, get_feature, copytree_with_merge, onWindows
+from .utils import (aslist, get_feature, copytree_with_merge, onWindows,
+                    add_sizes)
 
 
 class LogAsDebugFilter(logging.Filter):
@@ -415,6 +417,12 @@ def var_spool_cwl_detector(obj,           # type: Union[Dict, List, Text]
             r = var_spool_cwl_detector(value, obj, key) or r
     return r
 
+def eval_resource(builder, resource_req):  # type: (Builder, Text) -> Any
+        if expression.needs_parsing(resource_req):
+            visit_class(builder.job, ("File",), add_sizes)
+            return builder.do_eval(resource_req)
+        return resource_req
+
 
 class Process(six.with_metaclass(abc.ABCMeta, object)):
     def __init__(self, toolpath_object, **kwargs):
@@ -698,9 +706,9 @@ def evalResources(self, builder, kwargs):
             mn = None
             mx = None
             if resourceReq.get(a + "Min"):
-                mn = builder.do_eval(resourceReq[a + "Min"])
+                mn = eval_resource(builder, resourceReq[a + "Min"])
             if resourceReq.get(a + "Max"):
-                mx = builder.do_eval(resourceReq[a + "Max"])
+                mx = eval_resource(builder, resourceReq[a + "Max"])
             if mn is None:
                 mn = mx
             elif mx is None:

cwltool/utils.py

@@ -185,3 +185,15 @@ def bytes2str_in_dicts(a):
 
     # simply return elements itself
     return a
+
+def add_sizes(obj):  # type: (Dict[Text, Any]) -> None
+       if 'location' in obj:
+           try:
+               obj["size"] = os.stat(obj["location"][7:]).st_size  # strip off file://
+           except OSError:
+               pass
+       elif 'contents' in obj:
+               obj["size"] = len(obj['contents'])
+       else:
+           return  # best effort
+