Merge branch 'easybuilders:develop' into develop

Author: julianmorillo

easybuild/_deprecated.py

@@ -31,6 +31,7 @@
 """
 import contextlib
 import functools
+import json
 import os
 import re
 import signal
@@ -96,6 +97,12 @@ def cache_aware_func(cmd, *args, **kwargs):
     return cache_aware_func
 
 
+def json_loads(body):
+    """Deprecated wrapper for json.loads"""
+    _log.deprecated("json_loads is deprecated, use json.loads", '6.0')
+    return json.loads(body)
+
+
 def get_output_from_process(proc, read_size=None, asynchronous=False, print_deprecation_warning=True):
     """
     Get output from running process (that was opened with subprocess.Popen).

easybuild/base/fancylogger.py

@@ -578,8 +578,7 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'maxBytes': max_bytes,
         'backupCount': backup_count,
     }
-    if sys.version_info[0] >= 3:
-        handleropts['encoding'] = 'utf-8'
+    handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):

easybuild/framework/easyblock.py

@@ -98,9 +98,9 @@
 from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import (
-    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
-    MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
-    SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
+    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP,
+    MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP,
+    READY_STEP, SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
 )
 from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd
 from easybuild.tools.jenkins import write_to_xml
@@ -448,7 +448,7 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         if checksum and chksum_input_git is not None:
             # ignore any checksum for given filename due to changes in https://github.com/python/cpython/issues/90021
             # tarballs made for git repos are not reproducible when created with Python < 3.9
-            if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+            if sys.version_info < (3, 9):
                 print_warning(
                     "Reproducible tarballs of Git repos are only possible when using Python 3.9+ to run EasyBuild. "
                     f"Skipping checksum verification of {chksum_input} since Python < 3.9 is used."
@@ -2830,8 +2830,6 @@ def patch_step(self, beginpath=None, patches=None):
             self.log.info("Applying patch %s" % patch['name'])
             trace_msg("applying patch %s" % patch['name'])
 
-            # patch source at specified index (first source if not specified)
-            srcind = patch.get('source', 0)
             # if patch level is specified, use that (otherwise let apply_patch derive patch level)
             level = patch.get('level', None)
             # determine suffix of source path to apply patch in (if any)
@@ -2840,16 +2838,14 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
             options = patch.get('opts', None)  # Extra options for patch command
 
-            self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s; options: %s",
-                           srcind, level, srcpathsuffix, copy_patch, options)
+            self.log.debug("Patch level: %s; source path suffix: %s; copy patch: %s; options: %s",
+                           level, srcpathsuffix, copy_patch, options)
 
             if beginpath is None:
-                try:
-                    beginpath = self.src[srcind]['finalpath']
-                    self.log.debug("Determine begin path for patch %s: %s" % (patch['name'], beginpath))
-                except IndexError as err:
-                    raise EasyBuildError("Can't apply patch %s to source at index %s of list %s: %s",
-                                         patch['name'], srcind, self.src, err)
+                if not self.src:
+                    raise EasyBuildError("Can't apply patch %s to source if no sources are given", patch['name'])
+                beginpath = self.src[0]['finalpath']
+                self.log.debug("Determined begin path for patch %s: %s" % (patch['name'], beginpath))
             else:
                 self.log.debug("Using specified begin path for patch %s: %s" % (patch['name'], beginpath))
 
@@ -4662,18 +4658,19 @@ def run_step(self, step, step_methods):
         run_hook(step, self.hooks, pre_step_hook=True, args=[self])
 
         for step_method in step_methods:
+            # step_method is a lambda function that takes an EasyBlock instance as an argument,
+            # and returns the actual method
+            current_method = step_method(self)
             # Remove leading underscore from e.g. "_test_step"
-            method_name = '_'.join(step_method.__code__.co_names).lstrip('_')
+            method_name = current_method.__name__.lstrip('_')
             self.log.info("Running method %s part of step %s", method_name, step)
 
             if self.dry_run:
                 self.dry_run_msg("[%s method]", method_name)
 
                 # if an known possible error occurs, just report it and continue
                 try:
-                    # step_method is a lambda function that takes an EasyBlock instance as an argument,
-                    # and returns the actual method, so use () to execute it
-                    step_method(self)()
+                    current_method()
                 except Exception as err:
                     if build_option('extended_dry_run_ignore_errors'):
                         dry_run_warning("ignoring error %s" % err, silent=self.silent)
@@ -4682,9 +4679,7 @@ def run_step(self, step, step_methods):
                         raise
                 self.dry_run_msg('')
             else:
-                # step_method is a lambda function that takes an EasyBlock instance as an argument,
-                # and returns the actual method, so use () to execute it
-                step_method(self)()
+                current_method()
 
         run_hook(step, self.hooks, post_step_hook=True, args=[self])
 
@@ -5001,6 +4996,8 @@ def build_and_install_one(ecdict, init_env):
         _log.debug("Skip set to %s" % skip)
         app.cfg['skip'] = skip
 
+    hooks = load_hooks(build_option('hooks'))
+
     # build easyconfig
     error_msg = '(no error)'
     exit_code = None
@@ -5022,6 +5019,8 @@ def build_and_install_one(ecdict, init_env):
             else:
                 enabled_write_permissions = False
 
+        run_hook(EASYBLOCK, hooks, pre_step_hook=True, args=[app])
+
         result = app.run_all_steps(run_test_cases=run_test_cases)
 
         if not dry_run:
@@ -5127,6 +5126,8 @@ def ensure_writable_log_dir(log_dir):
             except EasyBuildError as err:
                 _log.warning("Unable to commit easyconfig to repository: %s", err)
 
+        run_hook(EASYBLOCK, hooks, post_step_hook=True, args=[app])
+
         # cleanup logs
         app.close_log()
 

easybuild/scripts/findPythonDeps.py

@@ -85,8 +85,7 @@ def can_run(cmd, *arguments):
 def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
     """Run the command and return the return code and output"""
     extra_args = kwargs or {}
-    if sys.version_info[0] >= 3:
-        extra_args['universal_newlines'] = True
+    extra_args['universal_newlines'] = True
     stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE
     p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args)
     out, err = p.communicate()

easybuild/tools/config.py

@@ -376,6 +376,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     EMPTY_LIST: [
         'accept_eula_for',
         'from_pr',
+        'ignore_pip_unversioned_pkgs',
         'include_easyblocks_from_pr',
         'robot',
         'search_paths',

easybuild/tools/filetools.py

@@ -137,7 +137,7 @@ def _hashlib_md5():
     to set usedforsecurity to False when supported (Python >= 3.9)
     """
     kwargs = {}
-    if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
+    if sys.version_info >= (3, 9):
         kwargs = {'usedforsecurity': False}
     return hashlib.md5(**kwargs)
 
@@ -219,10 +219,10 @@ def is_readable(path):
 
 
 def open_file(path, mode):
-    """Open a (usually) text file. If mode is not binary, then utf-8 encoding will be used for Python 3.x"""
+    """Open a (usually) text file. If mode is not binary, then utf-8 encoding will be used"""
     # This is required for text files in Python 3, especially until Python 3.7 which implements PEP 540.
     # This PEP opens files in UTF-8 mode if the C locale is used, see https://www.python.org/dev/peps/pep-0540
-    if sys.version_info[0] >= 3 and 'b' not in mode:
+    if 'b' not in mode:
         return open(path, mode, encoding='utf-8')
     else:
         return open(path, mode)
@@ -281,8 +281,8 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
 
     data_is_file_obj = hasattr(data, 'read')
 
-    # special care must be taken with binary data in Python 3
-    if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj):
+    # special care must be taken with binary data
+    if isinstance(data, bytes) or data_is_file_obj:
         mode += 'b'
 
     # don't bother showing a progress bar for small files (< 10MB)
@@ -418,7 +418,7 @@ def remove(paths):
     _log.info("Removing %d files & directories", len(paths))
 
     for path in paths:
-        if os.path.isfile(path):
+        if os.path.isfile(path) or os.path.islink(path):
             remove_file(path)
         elif os.path.isdir(path):
             remove_dir(path)

easybuild/tools/github.py

@@ -44,6 +44,7 @@
 import tempfile
 import time
 from datetime import datetime, timedelta
+from http.client import HTTPException
 from string import ascii_letters
 from urllib.request import HTTPError, URLError, urlopen
 
@@ -281,7 +282,7 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs):
 
     try:
         status, data = url.get(**kwargs)
-    except socket.gaierror as err:
+    except (socket.gaierror, HTTPException) as err:
         _log.warning("Error occurred while performing get request: %s", err)
         status, data = 0, None
 

easybuild/tools/hooks.py

@@ -62,6 +62,7 @@
 START = 'start'
 PARSE = 'parse'
 BUILD_AND_INSTALL_LOOP = 'build_and_install_loop'
+EASYBLOCK = 'easyblock'
 SINGLE_EXTENSION = 'single_extension'
 MODULE_WRITE = 'module_write'
 END = 'end'
@@ -86,6 +87,7 @@
     START,
     PARSE,
     PRE_PREF + BUILD_AND_INSTALL_LOOP,
+    PRE_PREF + EASYBLOCK,
 ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)]
      for p in [PRE_PREF, POST_PREF]] + [
     # pre-extensions hook is triggered before starting installation of extensions,
@@ -105,6 +107,7 @@
     POST_PREF + MODULE_STEP,
 ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:]
      for p in [PRE_PREF, POST_PREF]] + [
+    POST_PREF + EASYBLOCK,
     POST_PREF + BUILD_AND_INSTALL_LOOP,
     END,
     CANCEL,

easybuild/tools/options.py

@@ -498,6 +498,8 @@ def override_options(self):
             'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
             'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False),
             'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
+            'ignore-pip-unversioned-pkgs': ("List of installed unversioned (0.0.0) python packages to ignore in the "
+                                            "sanity check", 'strlist', 'store', []),
             'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
                                   None, 'store_true', False),
             'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),

easybuild/tools/py2vs3/py3.py

@@ -33,8 +33,6 @@
 """
 # these are not used here, but imported from here in other places
 import configparser  # noqa
-import json
-import sys
 import urllib.request as std_urllib  # noqa
 from collections.abc import Mapping  # noqa
 from functools import cmp_to_key
@@ -59,6 +57,7 @@
 except ImportError:
     HAVE_DISTUTILS = False
 
+from easybuild._deprecated import json_loads # noqa
 from easybuild.base.wrapper import mk_wrapper_baseclass  # noqa
 from easybuild.tools.run import subprocess_popen_text, subprocess_terminate  # noqa
 
@@ -75,18 +74,6 @@ def load_source(filename, path):
     return module
 
 
-def json_loads(body):
-    """Wrapper for json.loads that takes into account that Python versions older than 3.6 require a string value."""
-
-    if isinstance(body, bytes) and sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        # decode bytes string as regular string with UTF-8 encoding for Python 3.5.x and older
-        # only Python 3.6 and newer have support for passing bytes string to json.loads
-        # cfr. https://docs.python.org/2/library/json.html#json.loads
-        body = body.decode('utf-8', 'ignore')
-
-    return json.loads(body)
-
-
 def raise_with_traceback(exception_class, message, traceback):
     """Raise exception of specified class with given message and traceback."""
     raise exception_class(message).with_traceback(traceback)