Merge branch 'develop' into arch-specific-deps-for-java

Author: boegel

RELEASE_NOTES

@@ -3,6 +3,29 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.0.1 (October 15th 2019)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add 'parallel' to list of config templates (#3036)
+  - add GitHub Actions workflow to run easybuild-framework test suite (#3039)
+  - add 'retest' as a reason to --close-pr, to close/re-open PRs to trigger re-test in Travis (#3040)
+  - define $EB_SCRIPT_PATH in 'eb' wrapper script, and consider it before location of 'eb' determined via $PATH in get_paths_for function (#3046)
+  - add support for --remove-ghost-install-dirs configuration option, and warn about (potential) ghost install dirs by default when --force/--rebuild is used (#3050)
+- various bug fixes, including:
+  - update bootstrap script to support installing EasyBuild v4.0 (#3017)
+  - fix broken test_download_repo due to archiving of easyconfigs (#3019, #3023)
+  - avoid that --inject-checksums introduces list of patches for extensions as a single long line (#3025, #3034)
+  - enhance regex in fix_shebang method to fix more Python/Perl shebangs + avoid patching binary files (#3029)
+  - delete test gist that is created by --check-github (#3031)
+  - disable templates when defining easyconfig parameters in EasyConfig.set_keys() (#3037)
+  - avoid setting GC3Pie's max_in_flight to None if --job-max-jobs is not specified (#3038)
+  - fix use of obtain_file method for extensions (#3042)
+  - error out if some GC3Pie job failed (#3044)
+
+
 v4.0.0 (September 20th 2019)
 ----------------------------
 

easybuild/base/fancylogger.py

@@ -452,7 +452,9 @@ def getLogger(name=None, fname=False, clsname=False, fancyrecord=None):
     nameparts = []
 
     if not is_fancyroot():
-        nameparts.append(getRootLoggerName())
+        # deliberately not calling getRootLoggerName function to determine actual root logger name,
+        # because it is prohibitively expensive in some texts (even when using 'python -O')
+        nameparts.append('root')
 
     if fancyrecord is None:
         # Altough we could set it as default value in the function definition

easybuild/framework/easyblock.py

@@ -75,8 +75,8 @@
 from easybuild.tools.filetools import change_dir, convert_name, compute_checksum, copy_file, derive_alt_pypi_url
 from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
-from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_file, rmtree2
-from easybuild.tools.filetools import verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
+from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
@@ -453,8 +453,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
                 raise EasyBuildError('No file found for patch %s', patch_spec)
 
         if extension:
-            self.log.info("Fetched extension patches: %s" % patches)
-            return [patch['path'] for patch in patches]
+            self.log.info("Fetched extension patches: %s", patches)
+            return patches
         else:
             self.log.info("Added patches: %s" % self.patches)
 
@@ -544,6 +544,7 @@ def fetch_extension_sources(self, skip_checksums=False):
 
                                 if not skip_checksums:
                                     for patch in ext_patches:
+                                        patch = patch['path']
                                         # report both MD5 and SHA256 checksums,
                                         # since both are valid default checksum types
                                         for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
@@ -553,6 +554,7 @@ def fetch_extension_sources(self, skip_checksums=False):
                                     # verify checksum (if provided)
                                     self.log.debug('Verifying checksums for extension patches...')
                                     for idx, patch in enumerate(ext_patches):
+                                        patch = patch['path']
                                         checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
                                         if verify_checksum(patch, checksum):
                                             self.log.info('Checksum for extension patch %s verified', patch)
@@ -924,25 +926,19 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
         if os.path.exists(dir_name):
             self.log.info("Found old directory %s" % dir_name)
             if self.cfg['keeppreviousinstall']:
-                self.log.info("Keeping old directory %s (hopefully you know what you are doing)" % dir_name)
+                self.log.info("Keeping old directory %s (hopefully you know what you are doing)", dir_name)
                 return
-            elif clean:
-                try:
-                    rmtree2(dir_name)
-                    self.log.info("Removed old directory %s" % dir_name)
-                except OSError as err:
-                    raise EasyBuildError("Removal of old directory %s failed: %s", dir_name, err)
             elif build_option('module_only'):
                 self.log.info("Not touching existing directory %s in module-only mode...", dir_name)
+            elif clean:
+                remove_dir(dir_name)
+                self.log.info("Removed old directory %s", dir_name)
             else:
                 self.log.info("Moving existing directory %s out of the way...", dir_name)
-                try:
-                    timestamp = time.strftime("%Y%m%d-%H%M%S")
-                    backupdir = "%s.%s" % (dir_name, timestamp)
-                    shutil.move(dir_name, backupdir)
-                    self.log.info("Moved old directory %s to %s" % (dir_name, backupdir))
-                except OSError as err:
-                    raise EasyBuildError("Moving old directory to backup %s %s failed: %s", dir_name, backupdir, err)
+                timestamp = time.strftime("%Y%m%d-%H%M%S")
+                backupdir = "%s.%s" % (dir_name, timestamp)
+                move_file(dir_name, backupdir)
+                self.log.info("Moved old directory %s to %s", dir_name, backupdir)
 
         if dontcreateinstalldir:
             olddir = dir_name
@@ -1619,6 +1615,31 @@ def set_parallel(self):
         self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel'])
         self.log.info("Setting parallelism: %s" % self.cfg['parallel'])
 
+    def remove_module_file(self):
+        """Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""
+
+        if os.path.exists(self.mod_filepath):
+            # if installation directory used by module file differs from the one used now,
+            # either clean it up to avoid leaving behind a ghost installation, or warn about it
+            # (see also https://github.com/easybuilders/easybuild-framework/issues/3026)
+            old_installdir = self.module_generator.det_installdir(self.mod_filepath)
+
+            if old_installdir is None:
+                warning_msg = "Failed to determine installation directory from module file %s" % self.mod_filepath
+                warning_msg += ", can't clean up potential ghost installation for %s %s" % (self.name, self.version)
+                print_warning(warning_msg)
+
+            elif os.path.exists(old_installdir) and not os.path.samefile(old_installdir, self.installdir):
+                if build_option('remove_ghost_install_dirs'):
+                    remove_dir(old_installdir)
+                    self.log.info("Ghost installation directory %s removed", old_installdir)
+                    print_msg("Ghost installation directory %s removed", old_installdir)
+                else:
+                    print_warning("Likely ghost installation directory detected: %s", old_installdir)
+
+            self.log.info("Removing existing module file %s", self.mod_filepath)
+            remove_file(self.mod_filepath)
+
     #
     # STEP FUNCTIONS
     #
@@ -1669,9 +1690,7 @@ def check_readiness_step(self):
 
         # remove existing module file under --force (but only if --skip is not used)
         elif build_option('force') or build_option('rebuild'):
-            if os.path.exists(self.mod_filepath):
-                self.log.info("Removing existing module file %s", self.mod_filepath)
-                remove_file(self.mod_filepath)
+            self.remove_module_file()
 
     def fetch_step(self, skip_checksums=False):
         """Fetch source files and patches (incl. extensions)."""
@@ -3467,8 +3486,8 @@ def make_checksum_lines(checksums, indent_level):
                         print_msg(" * %s: %s" % (src_fn, checksum), log=_log)
                         ext_checksums.append((src_fn, checksum))
                     for ext_patch in ext.get('patches', []):
-                        patch_fn = os.path.basename(ext_patch)
-                        checksum = compute_checksum(ext_patch, checksum_type)
+                        patch_fn = os.path.basename(ext_patch['path'])
+                        checksum = compute_checksum(ext_patch['path'], checksum_type)
                         print_msg(" * %s: %s" % (patch_fn, checksum), log=_log)
                         ext_checksums.append((patch_fn, checksum))
 

easybuild/framework/easyconfig/easyconfig.py

@@ -67,7 +67,7 @@
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
 from easybuild.tools.modules import modules_tool
-from easybuild.tools.py2vs3 import OrderedDict, string_type
+from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
 from easybuild.tools.systemtools import check_os_dependency, get_cpu_architecture
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
@@ -530,12 +530,15 @@ def extend_params(self, extra, overwrite=True):
                 self.mandatory.append(key)
         self.log.debug("Updated list of mandatory easyconfig parameters: %s", self.mandatory)
 
-    def copy(self):
+    def copy(self, validate=None):
         """
         Return a copy of this EasyConfig instance.
         """
+        if validate is None:
+            validate = self.validation
+
         # create a new EasyConfig instance
-        ec = EasyConfig(self.path, validate=self.validation, hidden=self.hidden, rawtxt=self.rawtxt)
+        ec = EasyConfig(self.path, validate=validate, hidden=self.hidden, rawtxt=self.rawtxt)
         # take a copy of the actual config dictionary (which already contains the extra options)
         ec._config = copy.deepcopy(self._config)
         # since rawtxt is defined, self.path may not get inherited, make sure it does
@@ -2238,10 +2241,12 @@ def fix_deprecated_easyconfigs(paths):
     print_msg("\nAll done! Fixed %d easyconfigs (out of %d found).\n", fixed_cnt, cnt, prefix=False)
 
 
-class ActiveMNS(object):
-    """Wrapper class for active module naming scheme."""
+# singleton metaclass: only one instance is created
+BaseActiveMNS = create_base_metaclass('BaseActiveMNS', Singleton, object)
 
-    __metaclass__ = Singleton
+
+class ActiveMNS(BaseActiveMNS):
+    """Wrapper class for active module naming scheme."""
 
     def __init__(self, *args, **kwargs):
         """Initialize logger."""

easybuild/framework/easyconfig/tools.py

@@ -255,17 +255,39 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
     path_list.extend(sys.path)
 
     # figure out installation prefix, e.g. distutils install path for easyconfigs
-    eb_path = which('eb')
+
+    # prefer using path specified in $EB_SCRIPT_PATH (if defined), which is set by 'eb' wrapper script
+    eb_path = os.getenv('EB_SCRIPT_PATH')
+    if eb_path is None:
+        # try to determine location of 'eb' script via $PATH, as fallback mechanism
+        eb_path = which('eb')
+        _log.info("Location to 'eb' script (found via $PATH): %s", eb_path)
+    else:
+        _log.info("Found location to 'eb' script via $EB_SCRIPT_PATH: %s", eb_path)
+
     if eb_path is None:
         warning_msg = "'eb' not found in $PATH, failed to determine installation prefix!"
         _log.warning(warning_msg)
         print_warning(warning_msg)
     else:
-        # real location to 'eb' should be <install_prefix>/bin/eb
-        eb_path = resolve_path(eb_path)
+        # eb_path is location to 'eb' wrapper script, e.g. <install_prefix>/bin/eb
+        # so installation prefix is usually two levels up
         install_prefix = os.path.dirname(os.path.dirname(eb_path))
-        path_list.append(install_prefix)
-        _log.debug("Also considering installation prefix %s..." % install_prefix)
+
+        # only consider resolved path to 'eb' script if desired subdir is not found relative to 'eb' script location
+        if os.path.exists(os.path.join(install_prefix, 'easybuild', subdir)):
+            path_list.append(install_prefix)
+            _log.info("Also considering installation prefix %s (determined via path to 'eb' script)...", install_prefix)
+        else:
+            _log.info("Not considering %s (no easybuild/%s subdir found)", install_prefix, subdir)
+
+            # also consider fully resolved location to 'eb' wrapper
+            # see https://github.com/easybuilders/easybuild-framework/pull/2248
+            resolved_eb_path = resolve_path(eb_path)
+            if eb_path != resolved_eb_path:
+                install_prefix = os.path.dirname(os.path.dirname(resolved_eb_path))
+                path_list.append(install_prefix)
+                _log.info("Also considering installation prefix %s (via resolved path to 'eb')...", install_prefix)
 
     # look for desired subdirs
     for path in path_list:

easybuild/framework/extension.py

@@ -57,7 +57,7 @@ def __init__(self, mself, ext, extra_params=None):
         """
         self.master = mself
         self.log = self.master.log
-        self.cfg = self.master.cfg.copy()
+        self.cfg = self.master.cfg.copy(validate=False)
         self.ext = copy.deepcopy(ext)
         self.dry_run = self.master.dry_run
 

easybuild/framework/extensioneasyblock.py

@@ -32,7 +32,7 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extension import Extension
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import apply_patch, change_dir, extract_file
+from easybuild.tools.filetools import change_dir, extract_file
 from easybuild.tools.utilities import remove_unwanted_chars, trace_msg
 
 
@@ -88,6 +88,7 @@ def __init__(self, *args, **kwargs):
             self.installdir = self.master.installdir
             self.modules_tool = self.master.modules_tool
             self.module_generator = self.master.module_generator
+            self.robot_path = self.master.robot_path
             self.is_extension = True
             self.unpack_options = None
         else:
@@ -109,10 +110,7 @@ def run(self, unpack_src=False):
                 change_dir(self.start_dir)
 
         # patch if needed
-        if self.patches:
-            for patchfile in self.patches:
-                if not apply_patch(patchfile, self.ext_dir):
-                    raise EasyBuildError("Applying patch %s failed", patchfile)
+        EasyBlock.patch_step(self, beginpath=self.ext_dir)
 
     def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands=None):
         """

easybuild/tools/config.py

@@ -227,6 +227,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'module_only',
         'package',
         'read_only_installdir',
+        'remove_ghost_install_dirs',
         'rebuild',
         'robot',
         'rpath',

easybuild/tools/filetools.py

@@ -703,7 +703,7 @@ def find_eb_script(script_name):
         prev_script_loc = script_loc
 
         # fallback mechanism: check in location relative to location of 'eb'
-        eb_path = which('eb')
+        eb_path = os.getenv('EB_SCRIPT_PATH') or which('eb')
         if eb_path is None:
             _log.warning("'eb' not found in $PATH, failed to determine installation prefix")
         else: