macOS fixes (#1028)

set tmpdir_prefix in macOS-with-docker friendly way
normalize use of tmpdir_prefix

Author: mr-c

cwltool/command_line_tool.py

@@ -492,13 +492,15 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                           json_dumps(builder.bindings, indent=4))
         dockerReq, _ = self.get_requirement("DockerRequirement")
         if dockerReq is not None and runtimeContext.use_container:
-            out_prefix = getdefault(runtimeContext.tmp_outdir_prefix, 'tmp')
+            out_dir, out_prefix = os.path.split(
+                runtimeContext.tmp_outdir_prefix)
             j.outdir = runtimeContext.outdir or \
-                tempfile.mkdtemp(prefix=out_prefix)  # type: ignore
-            tmpdir_prefix = getdefault(runtimeContext.tmpdir_prefix, 'tmp')
+                tempfile.mkdtemp(prefix=out_prefix, dir=out_dir)
+            tmpdir_dir, tmpdir_prefix = os.path.split(
+                runtimeContext.tmpdir_prefix)
             j.tmpdir = runtimeContext.tmpdir or \
-                tempfile.mkdtemp(prefix=tmpdir_prefix)  # type: ignore
-            j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix)
+                tempfile.mkdtemp(prefix=tmpdir_prefix, dir=tmpdir_dir)
+            j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix, dir=tmpdir_dir)
         else:
             j.outdir = builder.outdir
             j.tmpdir = builder.tmpdir

cwltool/docker.py

@@ -231,7 +231,8 @@ def add_writable_file_volume(self,
                 # which is already going to be mounted
                 shutil.copy(volume.resolved, host_outdir_tgt)
             else:
-                tmpdir = tempfile.mkdtemp(dir=tmpdir_prefix)
+                tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
+                tmpdir = tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir)
                 file_copy = os.path.join(
                     tmpdir, os.path.basename(volume.resolved))
                 shutil.copy(volume.resolved, file_copy)
@@ -249,8 +250,9 @@ def add_writable_directory_volume(self,
         if volume.resolved.startswith("_:"):
             # Synthetic directory that needs creating first
             if not host_outdir_tgt:
+                tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                 new_dir = os.path.join(
-                    tempfile.mkdtemp(dir=tmpdir_prefix),
+                    tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                     os.path.basename(volume.target))
                 self.append_volume(runtime, new_dir, volume.target,
                                    writable=True)
@@ -262,7 +264,8 @@ def add_writable_directory_volume(self,
                                    writable=True)
             else:
                 if not host_outdir_tgt:
-                    tmpdir = tempfile.mkdtemp(dir=tmpdir_prefix)
+                    tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
+                    tmpdir = tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir)
                     new_dir = os.path.join(
                         tmpdir, os.path.basename(volume.resolved))
                     shutil.copytree(volume.resolved, new_dir)
@@ -355,7 +358,8 @@ def create_runtime(self,
                                   "please check it first")
                     exit(2)
             else:
-                cidfile_dir = tempfile.mkdtemp(dir=runtimeContext.tmpdir_prefix)
+                tmp_dir, tmp_prefix = os.path.split(runtimeContext.tmpdir_prefix)
+                cidfile_dir = tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir)
 
             cidfile_name = datetime.datetime.now().strftime("%Y%m%d%H%M%S-%f") + ".cid"
             if runtimeContext.cidfile_prefix is not None:

cwltool/job.py

@@ -519,8 +519,9 @@ def create_file_and_add_volume(self,
                                    ):  # type: (...) -> Text
         """Create the file and add a mapping."""
         if not host_outdir_tgt:
+            tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
             new_file = os.path.join(
-                tempfile.mkdtemp(dir=tmpdir_prefix),
+                tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                 os.path.basename(volume.resolved))
         writable = True if volume.type == "CreateWritableFile" else False
         if secret_store:
@@ -688,7 +689,8 @@ def docker_monitor(self, cidfile, tmpdir_prefix, cleanup_cidfile, process):
             except OSError:
                 cid = None
         max_mem = self.docker_get_memory(cid)
-        stats_file = tempfile.NamedTemporaryFile(dir=tmpdir_prefix)
+        tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
+        stats_file = tempfile.NamedTemporaryFile(prefix=tmp_prefix, dir=tmp_dir)
         with open(stats_file.name, mode="w") as stats_file_handle:
             stats_proc = subprocess.Popen(
                 ['docker', 'stats', '--no-trunc', '--format', '{{.MemPerc}}',

cwltool/main.py

@@ -754,6 +754,8 @@ def my_represent_none(self, data):  # pylint: disable=unused-argument
             default_mac_path = "/private/tmp/docker_tmp"
             if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX:
                 runtimeContext.tmp_outdir_prefix = default_mac_path
+            if runtimeContext.tmpdir_prefix == DEFAULT_TMP_PREFIX:
+                runtimeContext.tmpdir_prefix = default_mac_path
 
         for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"):
             if getattr(runtimeContext, dirprefix) and getattr(runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX:

cwltool/provenance.py

@@ -947,7 +947,9 @@ def __init__(self, temp_prefix_ro="tmp", orcid='', full_name=''):
         self.temp_prefix = temp_prefix_ro
         self.orcid = '' if not orcid else _valid_orcid(orcid)
         self.full_name = full_name
-        self.folder = os.path.abspath(tempfile.mkdtemp(prefix=temp_prefix_ro))  # type: Text
+        tmp_dir, tmp_prefix = os.path.split(temp_prefix_ro)
+        self.folder = os.path.abspath(tempfile.mkdtemp(prefix=tmp_prefix,
+            dir=tmp_dir))  # type: Text
         self.closed = False
         # map of filename "data/de/alsdklkas": 12398123 bytes
         self.bagged_size = {}  # type: Dict
@@ -1394,8 +1396,9 @@ def add_data_file(self, from_fp, timestamp=None, content_type=None):
         # type: (IO, Optional[datetime.datetime], Optional[str]) -> Text
         """Copy inputs to data/ folder."""
         self.self_check()
+        tmp_dir, tmp_prefix = os.path.split(self.temp_prefix)
         with tempfile.NamedTemporaryFile(
-                prefix=self.temp_prefix, delete=False) as tmp:
+                prefix=tmp_prefix, dir=tmp_dir, delete=False) as tmp:
             checksum = checksum_copy(from_fp, tmp)
 
         # Calculate hash-based file path

cwltool/singularity.py

@@ -202,8 +202,9 @@ def add_writable_file_volume(self,
                 runtime, volume.resolved, volume.target, writable=True)
             ensure_writable(volume.resolved)
         else:
+            tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
             file_copy = os.path.join(
-                tempfile.mkdtemp(dir=tmpdir_prefix),
+                tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                 os.path.basename(volume.resolved))
             shutil.copy(volume.resolved, file_copy)
             #volume.resolved = file_copy
@@ -221,8 +222,9 @@ def add_writable_directory_volume(self,
             if host_outdir_tgt is not None:
                 new_dir = host_outdir_tgt
             else:
+                tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                 new_dir = os.path.join(
-                    tempfile.mkdtemp(dir=tmpdir_prefix),
+                    tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                     os.path.basename(volume.resolved))
             os.makedirs(new_dir, 0o0755)
         else:
@@ -235,8 +237,9 @@ def add_writable_directory_volume(self,
                 ensure_writable(host_outdir_tgt)
             else:
                 if not self.inplace_update:
+                    tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                     dir_copy = os.path.join(
-                        tempfile.mkdtemp(dir=tmpdir_prefix),
+                        tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                         os.path.basename(volume.resolved))
                     shutil.copytree(volume.resolved, dir_copy)
                     source = dir_copy

setup.cfg

@@ -9,5 +9,5 @@ universal = 1
 test=pytest
 
 [tool:pytest]
-addopts=--ignore cwltool/schemas
+addopts=--ignore cwltool/schemas --basetemp ./tmp
 testpaths = tests

tests/test_ext.py

@@ -13,7 +13,7 @@
 import cwltool.process
 
 
-from .util import get_data, needs_docker, temp_dir, windows_needs_docker, get_main_output
+from .util import get_data, needs_docker, temp_dir, windows_needs_docker
 
 
 @needs_docker
@@ -55,8 +55,7 @@ def test_listing_v1_1():
 
 @needs_docker
 def test_double_overwrite(tmpdir):
-    try:
-        tmp = tempfile.mkdtemp()
+    with temp_dir() as tmp:
         tmp_name = os.path.join(tmp, "value")
 
         before_value, expected_value = "1", "3"
@@ -71,96 +70,76 @@ def test_double_overwrite(tmpdir):
             actual_value = f.read()
 
         assert actual_value == expected_value
-    finally:
-        shutil.rmtree(tmp)
 
 @needs_docker
 def test_disable_file_overwrite_without_ext():
-    try:
-        tmp = tempfile.mkdtemp()
-        out = tempfile.mkdtemp()
+    with temp_dir() as tmp:
+        with temp_dir() as out:
+            tmp_name = os.path.join(tmp, "value")
+            out_name = os.path.join(out, "value")
 
-        tmp_name = os.path.join(tmp, "value")
-        out_name = os.path.join(out, "value")
+            before_value, expected_value = "1", "2"
 
-        before_value, expected_value = "1", "2"
+            with open(tmp_name, "w") as f:
+                f.write(before_value)
 
-        with open(tmp_name, "w") as f:
-            f.write(before_value)
+            assert main(["--outdir", out, get_data('tests/wf/updateval.cwl'), "-r", tmp_name]) == 0
 
-        assert main(["--outdir", out, get_data('tests/wf/updateval.cwl'), "-r", tmp_name]) == 0
+            with open(tmp_name, "r") as f:
+                tmp_value = f.read()
+            with open(out_name, "r") as f:
+                out_value = f.read()
 
-        with open(tmp_name, "r") as f:
-            tmp_value = f.read()
-        with open(out_name, "r") as f:
-            out_value = f.read()
-
-        assert tmp_value == before_value
-        assert out_value == expected_value
-    finally:
-        shutil.rmtree(tmp)
-        shutil.rmtree(out)
+            assert tmp_value == before_value
+            assert out_value == expected_value
 
 @needs_docker
 def test_disable_dir_overwrite_without_ext():
-    try:
-        tmp = tempfile.mkdtemp()
-        out = tempfile.mkdtemp()
+    with temp_dir() as tmp:
+        with temp_dir() as out:
 
-        assert main(["--outdir", out, get_data('tests/wf/updatedir.cwl'), "-r", tmp]) == 0
+            assert main(["--outdir", out, get_data('tests/wf/updatedir.cwl'), "-r", tmp]) == 0
 
-        assert not os.listdir(tmp)
-        assert os.listdir(out)
-    finally:
-        shutil.rmtree(tmp)
-        shutil.rmtree(out)
+            assert not os.listdir(tmp)
+            assert os.listdir(out)
 
 @needs_docker
 def test_disable_file_creation_in_outdir_with_ext():
-    try:
-        tmp = tempfile.mkdtemp()
-        out = tempfile.mkdtemp()
+    with temp_dir() as tmp:
+        with temp_dir() as out:
 
-        tmp_name = os.path.join(tmp, "value")
-        out_name = os.path.join(out, "value")
+            tmp_name = os.path.join(tmp, "value")
+            out_name = os.path.join(out, "value")
 
-        before_value, expected_value = "1", "2"
+            before_value, expected_value = "1", "2"
 
-        with open(tmp_name, "w") as f:
-            f.write(before_value)
+            with open(tmp_name, "w") as f:
+                f.write(before_value)
 
-        params = ["--enable-ext", "--leave-outputs", "--outdir",
-                  out, get_data('tests/wf/updateval_inplace.cwl'), "-r", tmp_name]
-        assert main(params) == 0
+            params = ["--enable-ext", "--leave-outputs", "--outdir",
+                      out, get_data('tests/wf/updateval_inplace.cwl'), "-r", tmp_name]
+            assert main(params) == 0
 
-        with open(tmp_name, "r") as f:
-            tmp_value = f.read()
+            with open(tmp_name, "r") as f:
+                tmp_value = f.read()
 
-        assert tmp_value == expected_value
-        assert not os.path.exists(out_name)
-    finally:
-        shutil.rmtree(tmp)
-        shutil.rmtree(out)
+            assert tmp_value == expected_value
+            assert not os.path.exists(out_name)
 
 @needs_docker
 def test_disable_dir_creation_in_outdir_with_ext():
-    try:
-        tmp = tempfile.mkdtemp()
-        out = tempfile.mkdtemp()
-        params = ["--enable-ext", "--leave-outputs", "--outdir",
-                  out, get_data('tests/wf/updatedir_inplace.cwl'), "-r", tmp]
-        assert main(params) == 0
-
-        assert os.listdir(tmp)
-        assert not os.listdir(out)
-    finally:
-        shutil.rmtree(tmp)
-        shutil.rmtree(out)
+    with temp_dir() as tmp:
+        with temp_dir() as out:
+            params = ["--enable-ext", "--leave-outputs", "--outdir",
+                      out, get_data('tests/wf/updatedir_inplace.cwl'), "-r", tmp]
+            assert main(params) == 0
+
+            assert os.listdir(tmp)
+            assert not os.listdir(out)
 
 @needs_docker
 def test_write_write_conflict():
     with temp_dir('tmp') as tmp:
-        tmp = tempfile.mkdtemp()
         tmp_name = os.path.join(tmp, "value")
 
         before_value, expected_value = "1", "2"
@@ -178,7 +157,6 @@ def test_write_write_conflict():
 @pytest.mark.skip(reason="This test is non-deterministic")
 def test_read_write_conflict():
     with temp_dir('tmp') as tmp:
-        tmp = tempfile.mkdtemp()
         tmp_name = os.path.join(tmp, "value")
 
         with open(tmp_name, "w") as f: