Merge branch 'develop' into tests_python2

Author: boegel

easybuild/tools/filetools.py

@@ -65,7 +65,7 @@
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
 from easybuild.tools.config import build_option, install_path
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
-from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type
+from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg
 
 try:
@@ -1918,15 +1918,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None):
                 # climb up until we hit an existing path or the empty string (for relative paths)
                 while existing_parent_path and not os.path.exists(existing_parent_path):
                     existing_parent_path = os.path.dirname(existing_parent_path)
-                os.makedirs(path)
+                makedirs(path, exist_ok=True)
             else:
                 os.mkdir(path)
-        except FileExistsError as err:
-            if os.path.exists(path):
-                # This may happen if a parallel build creates the directory after we checked for its existence
-                _log.debug("Directory creation aborted as it seems it was already created: %s", err)
-            else:
-                raise EasyBuildError("Failed to create directory %s: %s", path, err)
         except OSError as err:
             raise EasyBuildError("Failed to create directory %s: %s", path, err)
 

easybuild/tools/py2vs3/py2.py

@@ -35,6 +35,7 @@
 import ConfigParser as configparser  # noqa
 import imp
 import json
+import os
 import subprocess
 import time
 import urllib2 as std_urllib  # noqa
@@ -115,3 +116,11 @@ def sort_looseversions(looseversions):
     # with Python 2, we can safely use 'sorted' on LooseVersion instances
     # (but we can't in Python 3, see https://bugs.python.org/issue14894)
     return sorted(looseversions)
+
+
+def makedirs(name, mode=0o777, exist_ok=False):
+    try:
+        os.makedirs(name, mode)
+    except OSError:
+        if not exist_ok or not os.path.isdir(name):
+            raise

easybuild/tools/py2vs3/py3.py

@@ -44,6 +44,7 @@
 from html.parser import HTMLParser  # noqa
 from itertools import zip_longest
 from io import StringIO  # noqa
+from os import makedirs  # noqa
 from string import ascii_letters, ascii_lowercase  # noqa
 from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen  # noqa
 from urllib.parse import urlencode  # noqa

test/framework/filetools.py

@@ -45,6 +45,7 @@
 from unittest import TextTestRunner
 from easybuild.tools import run
 import easybuild.tools.filetools as ft
+import easybuild.tools.py2vs3 as py2vs3
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
 from easybuild.tools.multidiff import multidiff
@@ -3389,6 +3390,17 @@ def test_set_gid_sticky_bits(self):
         self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
         self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
 
+    def test_compat_makedirs(self):
+        """Test compatibility layer for Python3 os.makedirs"""
+        name = os.path.join(self.test_prefix, 'folder')
+        self.assertNotExists(name)
+        py2vs3.makedirs(name)
+        self.assertExists(name)
+        # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3)
+        self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name)
+        py2vs3.makedirs(name, exist_ok=True)  # No error
+        self.assertExists(name)
+
     def test_create_unused_dir(self):
         """Test create_unused_dir function."""
         path = ft.create_unused_dir(self.test_prefix, 'folder')