Merge pull request #726 from common-workflow-language/more_wincompat

mr-c · web-flow · commit 0ec81f3d0990 · 2018-04-24T17:13:32.000+03:00
build on win-py36, &amp; 32bits
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+tests/wf/whale.txt test eol=lf
diff --git a/appveyor.yml b/appveyor.yml
@@ -14,28 +14,41 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python34"
+      PYTHON_VERSION: "3.4.x"
+      PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python35"
+      PYTHON_VERSION: "3.5.x"
+      PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
 
 install:
+  - "%PYTHON%\\python.exe -m pip install -U wheel setuptools pip pytest mock"
   # Get the latest stable version of Node.js or io.js
   - ps: Install-Product node $env:nodejs_version
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
 
 build_script:
-  - "%CMD_IN_ENV% python -m pip install -U setuptools pip"
-  - "%CMD_IN_ENV% pip install pytest mock"
-  - "%CMD_IN_ENV% pip install ."
+  - "%PYTHON%\\python.exe -m pip install ."
 
 
 test_script:
-  - "%CMD_IN_ENV% py.test --verbose -p no:cacheprovider"
+  - "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider"
 
 branches:
   only:
diff --git a/cwltool/utils.py b/cwltool/utils.py
@@ -3,6 +3,7 @@
 # no imports from cwltool allowed
 
 import os
+import platform
 import shutil
 import stat
 from typing import Any, Callable, Dict, List, Text, Tuple, Union
@@ -60,15 +61,24 @@ def copytree_with_merge(src, dst, symlinks=False, ignore=None):
             shutil.copy2(s, d)
 
 
-# changes windowspath(only) appropriately to be passed to docker run command
-# as docker treat them as unix paths so convert C:\Users\foo to /C/Users/foo
 def docker_windows_path_adjust(path):
     # type: (Text) -> (Text)
+    r"""
+    Changes only windows paths so that the can be appropriately passed to the
+    docker run command as as docker treats them as unix paths.
+
+    Example: 'C:\Users\foo to /C/Users/foo (Docker for Windows) or /c/Users/foo
+    (Docker toolbox).
+    """
     if path is not None and onWindows():
-        sp=path.split(':')
-        if len(sp)==2:
-            sp[0]=sp[0].capitalize()  # Capitalizing windows Drive letters
-            path=':'.join(sp)
+        split = path.split(':')
+        if len(split) == 2:
+            if platform.win32_ver()[0] in ('7', '8'):  # type: ignore
+                split[0] = split[0].lower()  # Docker toolbox uses lowecase windows Drive letters
+            else:
+                split[0] = split[0].capitalize()
+                # Docker for Windows uses uppercase windows Drive letters
+            path = ':'.join(split)
         path = path.replace(':', '').replace('\\', '/')
         return path if path[0] == '/' else '/' + path
     return path
diff --git a/tests/test_check.py b/tests/test_check.py
@@ -8,14 +8,12 @@
 import cwltool.workflow
 import pytest
 from cwltool.main import main
-from cwltool.utils import onWindows
 
-from .util import get_data
+from .util import get_data, needs_docker
 
 
 class TestCheck(unittest.TestCase):
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
+    @needs_docker
     def test_output_checking(self):
         self.assertEquals(main([get_data('tests/wf/badout1.cwl')]), 1)
         self.assertEquals(main([get_data('tests/wf/badout2.cwl')]), 1)
diff --git a/tests/test_examples.py b/tests/test_examples.py
@@ -8,7 +8,6 @@
 from io import StringIO
 
 from cwltool.errors import WorkflowException
-from cwltool.utils import onWindows
 
 try:
     reload
@@ -25,8 +24,9 @@
 import cwltool.workflow
 import schema_salad.validate
 from cwltool.main import main
+from cwltool.utils import onWindows
 
-from .util import get_data
+from .util import get_data, needs_docker
 
 sys.argv = ['']
 
@@ -603,9 +603,7 @@ def test_no_js_console(self):
             self.assertNotIn("[err] Error message", stderr)
 
 
-@pytest.mark.skipif(onWindows(),
-                    reason="Instance of cwltool is used, on Windows it invokes a default docker container"
-                           "which is not supported on AppVeyor")
+@needs_docker
 class TestCache(TestCmdLine):
     def test_wf_without_container(self):
         test_file = "hello-workflow.cwl"
@@ -614,13 +612,12 @@ def test_wf_without_container(self):
         self.assertIn("completed success", stderr)
         self.assertEquals(error_code, 0)
 
-@pytest.mark.skipif(onWindows(),
-                    reason="Instance of cwltool is used, on Windows it invokes a default docker container"
-                           "which is not supported on AppVeyor")
+@needs_docker
 class TestChecksum(TestCmdLine):
 
     def test_compute_checksum(self):
-        f = cwltool.factory.Factory(compute_checksum=True, use_container=False)
+        f = cwltool.factory.Factory(compute_checksum=True,
+                use_container=onWindows())
         echo = f.make(get_data("tests/wf/cat-tool.cwl"))
         output = echo(file1={
                 "class": "File",
diff --git a/tests/test_ext.py b/tests/test_ext.py
@@ -11,11 +11,10 @@
 import cwltool.workflow
 from cwltool.main import main
 from cwltool.utils import onWindows
-from .util import get_data
+from .util import get_data, needs_docker
 
 
-@pytest.mark.skipif(onWindows(),
-                    reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
+@needs_docker
 class TestListing(unittest.TestCase):
     def test_missing_enable_ext(self):
         # Require that --enable-ext is provided.
diff --git a/tests/test_override.py b/tests/test_override.py
@@ -8,15 +8,13 @@
 import pytest
 import json
 from cwltool.main import main
-from cwltool.utils import onWindows
 from six import StringIO
 
-from .util import get_data
+from .util import get_data, needs_docker
 
 
 class TestOverride(unittest.TestCase):
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
+    @needs_docker
     def test_overrides(self):
         sio = StringIO()
 
diff --git a/tests/test_pack.py b/tests/test_pack.py
@@ -17,8 +17,7 @@
 from cwltool.load_tool import fetch_document, validate_document
 from cwltool.main import makeRelative, main, print_pack
 from cwltool.pathmapper import adjustDirObjs, adjustFileObjs
-from cwltool.utils import onWindows
-from .util import get_data
+from .util import get_data, needs_docker
 
 
 class TestPack(unittest.TestCase):
@@ -96,9 +95,7 @@ def _pack_idempotently(self, document):
         double_packed = json.loads(print_pack(document_loader, processobj, uri2, metadata))
         self.assertEqual(packed, double_packed)
 
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of cwltool is used, on Windows it invokes a default docker container"
-                               "which is not supported on AppVeyor")
+    @needs_docker
     def test_packed_workflow_execution(self):
         load_tool.loaders = {}
         test_wf = "tests/wf/count-lines1-wf.cwl"
@@ -108,7 +105,7 @@ def test_packed_workflow_execution(self):
         document_loader, avsc_names, processobj, metadata, uri = validate_document(
             document_loader, workflowobj, uri)
         packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
-        temp_packed_path = tempfile.mkstemp()[1]
+        temp_packed_handle, temp_packed_path = tempfile.mkstemp()
         with open(temp_packed_path, 'w') as f:
             json.dump(packed, f)
         normal_output = StringIO()
@@ -120,11 +117,10 @@ def test_packed_workflow_execution(self):
                                 get_data(test_wf_job)],
                                stdout=normal_output), 0)
         self.assertEquals(json.loads(packed_output.getvalue()), json.loads(normal_output.getvalue()))
+        os.close(temp_packed_handle)
         os.remove(temp_packed_path)
 
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of cwltool is used, on Windows it invokes a default docker container"
-                               "which is not supported on AppVeyor")
+    @needs_docker
     def test_preserving_namespaces(self):
         test_wf = "tests/wf/formattest.cwl"
         test_wf_job = "tests/wf/formattest-job.json"
@@ -134,7 +130,7 @@ def test_preserving_namespaces(self):
             document_loader, workflowobj, uri)
         packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
         assert "$namespaces" in packed
-        temp_packed_path = tempfile.mkstemp()[1]
+        temp_packed_handle, temp_packed_path = tempfile.mkstemp()
         with open(temp_packed_path, 'w') as f:
             json.dump(packed, f)
         normal_output = StringIO()
@@ -146,4 +142,5 @@ def test_preserving_namespaces(self):
                                 get_data(test_wf_job)],
                                stdout=normal_output), 0)
         self.assertEquals(json.loads(packed_output.getvalue()), json.loads(normal_output.getvalue()))
+        os.close(temp_packed_handle)
         os.remove(temp_packed_path)
diff --git a/tests/test_relax_path_checks.py b/tests/test_relax_path_checks.py
@@ -3,8 +3,8 @@
 import pytest
 from tempfile import NamedTemporaryFile
 
+from .util import needs_docker
 from cwltool.main import main
-from cwltool.utils import onWindows
 
 
 class ToolArgparse(unittest.TestCase):
@@ -26,8 +26,7 @@ class ToolArgparse(unittest.TestCase):
 baseCommand: [cat]
 '''
 
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
+    @needs_docker
     def test_spaces_in_input_files(self):
         with NamedTemporaryFile(mode='w', delete=False) as f:
             f.write(self.script)
diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py
@@ -4,9 +4,8 @@
 from tempfile import NamedTemporaryFile
 
 from cwltool.main import main
-from cwltool.utils import onWindows
 
-from .util import get_data
+from .util import get_data, needs_docker
 
 class ToolArgparse(unittest.TestCase):
     script = '''
@@ -66,8 +65,7 @@ class ToolArgparse(unittest.TestCase):
 outputs: []
 '''
 
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
+    @needs_docker
     def test_help(self):
         with NamedTemporaryFile(mode='w', delete=False) as f:
             f.write(self.script)
@@ -79,8 +77,7 @@ def test_help(self):
                 get_data('tests/echo.cwl')]), 0)
 
 
-    @pytest.mark.skipif(onWindows(),
-                        reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
+    @needs_docker
     def test_bool(self):
         with NamedTemporaryFile(mode='w', delete=False) as f:
             f.write(self.script2)
diff --git a/tests/util.py b/tests/util.py
@@ -3,7 +3,8 @@
 
 from pkg_resources import (Requirement, ResolutionError,  # type: ignore
                            resource_filename)
-
+import distutils.spawn
+import pytest
 
 def get_data(filename):
     filename = os.path.normpath(
@@ -16,4 +17,12 @@ def get_data(filename):
         pass
     if not filepath or not os.path.isfile(filepath):
         filepath = os.path.join(os.path.dirname(__file__), os.pardir, filename)
+        # warning, __file__ is all lowercase on Windows systems, this can
+        # sometimes conflict with docker toolkit. Workaround: pip install .
+        # and run the tests elsewhere via python -m pytest --pyarg cwltool
     return filepath
+
+
+needs_docker = pytest.mark.skipif(not bool(distutils.spawn.find_executable('docker')),
+                                  reason="Requires the docker executable on the "
+                                  "system path.")
diff --git a/tests/wf/cat-tool.cwl b/tests/wf/cat-tool.cwl
@@ -3,6 +3,10 @@
 class: CommandLineTool
 cwlVersion: v1.0
 
+hints:
+  DockerRequirement:
+    dockerPull: frolvlad/alpine-bash
+
 inputs:
   file1: File
 
diff --git a/windowsdoc.md b/windowsdoc.md
@@ -1,17 +1,19 @@
 # Windows Compatibility
-cwltool is compatible with Windows. This means that you can create and run CWL
-workflows on Windows. On Windows, all workflows and tools are executed in
-[Docker Containers](https://docs.docker.com/docker-for-windows/). The default
-Docker Container is
+The CWL reference runner, cwltool, is compatible with Microsoft Windows when
+Docker is installed. On Windows, all CWL CommandLineTools are executed using
+[Docker software containers](https://docs.docker.com/docker-for-windows/). The
+default Docker Container is
 [Alpine with Bash support](https://github.com/frol/docker-alpine-bash). You can
 specify other Docker Containers for your tools and workflows using hints,
 [requirements](http://www.commonwl.org/v1.0/CommandLineTool.html#DockerRequirement)),
 or the `--default-container` cwltool option.
 
 ## Supported Windows versions
 * Windows 10 with native [Docker for Windows](https://docs.docker.com/docker-for-windows/).
-* Windows 8.1 with [Docker ToolBox](https://docs.docker.com/toolbox/toolbox_install_windows/).
-* Windows 7 & 8 with Docker ToolBox may work (Not tested, please let us know!).
+* Windows 7, 8, and 8.1 with [Docker ToolBox](https://docs.docker.com/toolbox/toolbox_install_windows/).
+
+If you are using Docker Toolbox, then you must run cwltool in the Docker
+Quickstart Terminal.
 
 ## Installation
 
@@ -26,7 +28,7 @@ Before installing cwltool, please install:
 * [Node.js](https://nodejs.org/en/download/) (optional, please install if your
   workflows or tools contain [Javascript Expressions](http://www.commonwl.org/v1.0/CommandLineTool.html#InlineJavascriptRequirement))
 
-### Install using pip
+### Install using pip (recommended)
 
 ```
 pip install cwltool
@@ -58,18 +60,25 @@ There are two types of tests available for cwltool: unit tests and conformance t
 
 ### Unit tests
 
-To run cwltool's unit tests, go to the cwltool repository on your system and run:
+To run cwltool's unit tests, run the following command:
+```
+python -m pytest --pyarg cwltool
+```
+
+Or go to the checkout of the cwltool Git repository on your system and run:
 
 ```
 python setup.py test
 ```
 
+
+
 ### Conformance tests
 
-To run the conformance tests, follow these instructions:
+To run the CWL conformance tests, follow these instructions:
 
 ```
-pip install cwltest  
+pip install cwltest mock 
 git clone https://github.com/common-workflow-language/common-workflow-language.git   
 cd common-workflow-language/v1.0  
 cwltest --test conformance_test_v1.0.yaml -j 4 --tool cwltool
@@ -130,3 +139,4 @@ can't then use the `--eval-timeout` argument and set a higher timeout value.
 *If you still have problems with setting up and using Docker on Windows, please
 consult the online Docker Community. If the problem is specific to cwltool,
 create an [issue on cwltool](https://github.com/common-workflow-language/cwltool/issues).*
+
