Merge pull request #4294 from easybuilders/4.8.x

release EasyBuild v4.8.0

Author: boegel

.github/workflows/container_tests.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [2.7, 3.7]
+        python: [3.7]
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/container_tests_apptainer.yml

@@ -14,14 +14,14 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [2.7, 3.7]
+        python: [3.7]
         apptainer: [1.0.0, 1.1.7]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/eb_command.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/linting.yml

@@ -13,13 +13,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/unit_tests.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.6]
+        python: [3.6]
         modules_tool:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
@@ -81,7 +81,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -207,10 +207,6 @@ jobs:
           # create file owned by root but writable by anyone (used by test_copy_file)
           sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
           sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-          # silence deprecation warning when using Python 2, since it breaks a bunch of tests
-          if [ "${{matrix.python}}" == '2.7' ]; then
-            export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2
-          fi
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)

RELEASE_NOTES

@@ -4,6 +4,22 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.0 (7 July 2023)
+--------------------
+
+feature release
+
+- various enhancements, including:
+  - use version suffixes in PR title instead of Python versions when using --new-pr (#4253 + #4286)
+  - add script to find updated ECs for installed module (#4271)
+- various bug fixes, including:
+  - fix regex for extracting glibc version from output of '`ldd --version`' in Gentoo Linux (#4290)
+  - allow versionsuffix to be set to None in `det_full_ec_version` (#4292)
+- other changes:
+  - stop running tests with Python 2.7 since it is no longer supported in GitHub Action (#4267)
+  - replace imp.load_source by leveraging importlib.util.module_from_spec when using Python 3 (#4280)
+
+
 v4.7.2 (27 May 2023)
 --------------------
 

easybuild/scripts/findUpdatedEcs.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[0;33m'
+NC='\033[0m'
+
+function printError {
+    echo -e "${RED}$@${NC}"
+}
+
+verbose=0
+function printVerbose {
+    (( verbose == 0 )) || echo "$@"
+}
+
+function checkModule {
+    moduleFolder="$1"
+    moduleName="$(basename "$(dirname "$moduleFolder")")"
+    moduleVersion="$(basename "$moduleFolder")"
+    moduleStr="$moduleName/$moduleVersion"
+    printVerbose "Processing $moduleStr"
+    ec_glob=( "$moduleFolder/easybuild/"*.eb )
+    if [[ ! -e "${ec_glob[@]}" ]]; then
+        printError "=== Did not find installed EC for $moduleStr"
+        return
+    fi
+    ec_installed="$ec_glob"
+    ec_filename=$(basename "$ec_installed")
+    # Try with most likely location first for speed
+    first_letter=${ec_filename:0:1}
+    letterPath=$easyconfigFolder/${first_letter,,}
+    if [[ -d "$letterPath" ]]; then
+      ec_new="$(find "$letterPath" -type f -name "$ec_filename")"
+    else
+      ec_new=
+    fi
+    # Fallback if not found
+    [[ -n "$ec_new" ]] || ec_new="$(find "$easyconfigFolder" -type f -name "$ec_filename")"
+    if [[ -z "$ec_new" ]]; then
+      printError "=== Did not find new EC $ec_filename"
+    elif [[ ! -e "$ec_new" ]]; then
+        printError "=== Found multiple new ECs: $ec_new"
+    elif ! out=$(diff "$ec_installed" "$ec_new"); then
+        echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}"
+        if ((showDiff == 1)); then
+            echo "$out"
+        fi
+    fi
+}
+
+ecDefaultFolder=
+if path=$(which eb 2>/dev/null); then
+  path=$(dirname "$path")
+  for p in "$path" "$(dirname "$path")"; do
+    if [ -d "$p/easybuild/easyconfigs" ]; then
+      ecDefaultFolder=$p
+      break
+    fi
+  done
+fi
+
+function usage {
+  echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER"
+  echo
+  echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed."
+  echo "Can either check the currently loaded modules or all modules installed in a specific location"
+  echo
+  echo "--verbose                Verbose status output while checking"
+  echo "--loaded                 Check only currently loaded modules"
+  echo "--diff                   Show diff of changed module files"
+  echo "--modules INSTALLPATH    Check all modules in the specified (software) installpath, i.e. the root of module-binaries"
+  echo "--easyconfigs EC-FOLDER  Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}"
+  exit 0
+}
+
+checkLoadedModules=0
+showDiff=0
+modulesFolder=""
+easyconfigFolder=$ecDefaultFolder
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -h|--help)
+            usage;;
+        -v|--verbose)
+            verbose=1;;
+        -d|--diff)
+            showDiff=1;;
+        -l|--loaded)
+            checkLoadedModules=1;;
+        -m|--modules)
+            modulesFolder="$2"
+            shift;;
+        -e|--easyconfigs)
+            easyconfigFolder="$2"
+            shift;;
+        *)
+            printError "Unknown argument: $1"
+            exit 1;;
+    esac
+    shift
+done
+
+if [ -z "$easyconfigFolder" ]; then
+    printError "Folder to easyconfigs not given!" && exit 1
+fi
+if [ -z "$modulesFolder" ]; then
+    if (( checkLoadedModules == 0 )); then
+        printError "Need either --modules or --loaded to specify what to check!" && exit 1
+    fi
+elif (( checkLoadedModules == 1 )); then
+    printError "Cannot specify --modules and --loaded!" && exit 1
+fi
+
+if [ -d "$easyconfigFolder/easybuild/easyconfigs" ]; then
+    easyconfigFolder="$easyconfigFolder/easybuild/easyconfigs"
+fi
+
+if (( checkLoadedModules == 1 )); then
+    for varname in $(compgen -A variable | grep '^EBROOT'); do
+        checkModule "${!varname}"
+    done
+else
+    for module in "$modulesFolder"/*/*/easybuild; do
+        checkModule "$(dirname "$module")"
+    done
+fi

easybuild/tools/filetools.py

@@ -44,7 +44,6 @@
 import difflib
 import glob
 import hashlib
-import imp
 import inspect
 import itertools
 import os
@@ -66,7 +65,7 @@
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
 from easybuild.tools.config import build_option, install_path
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
-from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
+from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg
 
 try:
@@ -2801,7 +2800,7 @@ def install_fake_vsc():
 def get_easyblock_class_name(path):
     """Make sure file is an easyblock and get easyblock class name"""
     fn = os.path.basename(path).split('.')[0]
-    mod = imp.load_source(fn, path)
+    mod = load_source(fn, path)
     clsmembers = inspect.getmembers(mod, inspect.isclass)
     for cn, co in clsmembers:
         if co.__module__ == mod.__name__:

easybuild/tools/github.py

@@ -1581,6 +1581,45 @@ def new_branch_github(paths, ecs, commit_msg=None):
     return res
 
 
+def det_pr_title(ecs):
+    """
+    Create title for PR based on first easyconfigs
+    :param ecs: list of parsed easyconfigs
+    """
+
+    # only use most common toolchain(s) in toolchain label of PR title
+    toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in ecs]
+    toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)])
+    toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]])
+
+    # only use most common module class(es) in moduleclass label of PR title
+    classes = [ec['moduleclass'] for ec in ecs]
+    classes_counted = sorted([(classes.count(c), c) for c in nub(classes)])
+    class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]])
+
+    names_and_versions = nub(["%s v%s" % (ec.name, ec.version) for ec in ecs])
+    if len(names_and_versions) <= 3:
+        main_title = ', '.join(names_and_versions)
+    else:
+        main_title = ', '.join(names_and_versions[:3] + ['...'])
+
+    title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title)
+
+    # Find all suffixes
+    suffixes = []
+    for ec in ecs:
+        if 'versionsuffix' in ec and ec['versionsuffix']:
+            suffixes.append(ec['versionsuffix'].strip('-').replace('-', ' '))
+    if suffixes:
+        suffixes = sorted(nub(suffixes))
+        if len(suffixes) <= 2:
+            title += ' w/ ' + ', '.join(suffixes)
+        else:
+            title += ' w/ ' + ', '.join(suffixes[:2] + ['...'])
+
+    return title
+
+
 @only_if_module_is_available('git', pkgname='GitPython')
 def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, pr_metadata=None, commit_msg=None):
     """
@@ -1691,42 +1730,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
 
     labels = det_pr_labels(file_info, pr_target_repo)
 
-    if pr_target_repo == GITHUB_EASYCONFIGS_REPO:
-        # only use most common toolchain(s) in toolchain label of PR title
-        toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']]
-        toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)])
-        toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]])
-
-        # only use most common module class(es) in moduleclass label of PR title
-        classes = [ec['moduleclass'] for ec in file_info['ecs']]
-        classes_counted = sorted([(classes.count(c), c) for c in nub(classes)])
-        class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]])
-
     if title is None:
         if pr_target_repo == GITHUB_EASYCONFIGS_REPO:
             if file_info['ecs'] and all(file_info['new']) and not deleted_paths:
-                # mention software name/version in PR title (only first 3)
-                names_and_versions = nub(["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']])
-                if len(names_and_versions) <= 3:
-                    main_title = ', '.join(names_and_versions)
-                else:
-                    main_title = ', '.join(names_and_versions[:3] + ['...'])
-
-                title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title)
-
-                # if Python is listed as a dependency, then mention Python version(s) in PR title
-                pyver = []
-                for ec in file_info['ecs']:
-                    # iterate over all dependencies (incl. build dependencies & multi-deps)
-                    for dep in ec.dependencies():
-                        if dep['name'] == 'Python':
-                            # check whether Python is listed as a multi-dep if it's marked as a build dependency
-                            if dep['build_only'] and 'Python' not in ec['multi_deps']:
-                                continue
-                            else:
-                                pyver.append(dep['version'])
-                if pyver:
-                    title += " w/ Python %s" % ' + '.join(sorted(nub(pyver)))
+                title = det_pr_title(file_info['ecs'])
         elif pr_target_repo == GITHUB_EASYBLOCKS_REPO:
             if file_info['eb_names'] and all(file_info['new']) and not deleted_paths:
                 plural = 's' if len(file_info['eb_names']) > 1 else ''

easybuild/tools/hooks.py

@@ -30,10 +30,10 @@
 * Kenneth Hoste (Ghent University)
 """
 import difflib
-import imp
 import os
 
 from easybuild.base import fancylogger
+from easybuild.tools.py2vs3 import load_source
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
 
@@ -123,7 +123,7 @@ def load_hooks(hooks_path):
                 _log.info("Importing hooks implementation from %s...", hooks_path)
                 try:
                     # import module that defines hooks, and collect all functions of which name ends with '_hook'
-                    imported_hooks = imp.load_source(hooks_filename, hooks_path)
+                    imported_hooks = load_source(hooks_filename, hooks_path)
                     for attr in dir(imported_hooks):
                         if attr.endswith(HOOK_SUFF):
                             hook = getattr(imported_hooks, attr)