Merge branch 'develop' into optarch-map

Author: Flamefire

.github/workflows/container_tests.yml

@@ -17,18 +17,18 @@ jobs:
         python: [3.7]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
 
     - name: install OS & Python packages
       run: |
         # for modules tool
-        APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+        APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
         # for building Singularity images
         APT_PKGS+=" rpm dnf"
 

.github/workflows/container_tests_apptainer.yml

@@ -15,13 +15,13 @@ jobs:
     strategy:
       matrix:
         python: [3.7]
-        apptainer: [1.0.0, 1.1.7]
+        apptainer: [1.0.0, 1.1.7, 1.3.6]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -31,7 +31,7 @@ jobs:
         # for building CentOS 7 container images
         APT_PKGS="rpm dnf"
         # for modules tool
-        APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+        APT_PKGS+=" lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
 
         # Avoid apt-get update, as we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
@@ -41,12 +41,6 @@ jobs:
           sudo apt-get install $APT_PKGS
         fi
 
-        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
-        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
-            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
-        fi
-
     - name: install Lmod
       run: |
           # avoid downloading modules tool sources into easybuild-framework dir

.github/workflows/eb_command.yml

@@ -11,16 +11,19 @@ concurrency:
 
 jobs:
   test-eb:
-    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13']
+        include:
+          - python: 3.7
+            os: ubuntu-22.04
       fail-fast: false
+    runs-on: ${{matrix.os || 'ubuntu-24.04'}}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -32,9 +35,13 @@ jobs:
         # update to latest pip, check version
         pip install --upgrade pip
         pip --version
+        if ! python -c "import distutils" 2> /dev/null; then
+            # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+            pip install --upgrade setuptools
+        fi
 
         # for modules tool
-        APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+        APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
 
         # Avoid apt-get update, as we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
@@ -44,12 +51,6 @@ jobs:
           sudo apt-get install $APT_PKGS
         fi
 
-        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
-        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
-            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
-        fi
-
     - name: install modules tool
       run: |
           # avoid downloading modules tool sources into easybuild-framework dir
@@ -91,7 +92,7 @@ jobs:
           pymajver=$(python -c 'import sys; print(sys.version_info[0])')
           pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
           # check patterns in verbose output
-          for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+          for pattern in "^>> Considering .python3.\.\.\." "^>> .python3. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python3' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python3 \(.*/bin/python3\)" "^This is EasyBuild 5\.[0-9.]\+"; do
               echo "Looking for pattern \"${pattern}\" in eb_version.out..."
               grep "$pattern" eb_version.out
           done
@@ -103,7 +104,7 @@ jobs:
           for eb_python in "python${pymajver}" "python${pymajminver}"; do
               export EB_PYTHON="${eb_python}"
               eb --version | tee eb_version.out 2>&1
-              for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+              for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 5\.[0-9.]\+"; do
                   echo "Looking for pattern \"${pattern}\" in eb_version.out..."
                   grep "$pattern" eb_version.out
               done

.github/workflows/end2end.yml

@@ -7,20 +7,20 @@ jobs:
     strategy:
       matrix:
         container:
-          - centos-7.9
           - centos-8.5
-          - fedora-36
+          - fedora-41
           - opensuse-15.4
-          - rockylinux-8.8
-          - rockylinux-9.2
+          - rockylinux-8.10
+          - rockylinux-9.5
           - ubuntu-20.04
           - ubuntu-22.04
+          - ubuntu-24.04
       fail-fast: false
     container:
       image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
       - name: download and unpack easyblocks and easyconfigs repositories
         run: |
@@ -49,6 +49,7 @@ jobs:
             "eb --show-system-info"
             "eb --check-eb-deps"
             "eb --show-config"
+            "eb -x bzip2-1.0.8.eb"
           )
           for cmd in "${cmds[@]}"; do
               echo ">>> $cmd"
@@ -58,4 +59,8 @@ jobs:
       - name: End-to-end test of installing bzip2 with EasyBuild
         shell: bash
         run: |
-          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"
+          EB_ARGS=''
+          if [[ "${{ matrix.container }}" == "fedora-41" ]] || [[ "${{ matrix.container }}" == "ubuntu-24.04" ]]; then
+              EB_ARGS='--filter-deps=binutils'
+          fi
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}"

.github/workflows/linting.yml

@@ -10,16 +10,19 @@ concurrency:
 
 jobs:
   python-linting:
-    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
-        
+        python: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13']
+        include:
+          - python: 3.7
+            os: ubuntu-22.04
+      fail-fast: false
+    runs-on: ${{matrix.os || 'ubuntu-24.04'}}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -30,10 +33,4 @@ jobs:
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: |
-        # don't check py2vs3/py3.py when testing with Python 2, and vice versa
-        if [[ "${{ matrix.python-version }}" =~ "2." ]]; then
-            py_excl=py3
-        else
-            py_excl=py2
-        fi
-        flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py
+        flake8

.github/workflows/unit_tests.yml

@@ -11,61 +11,55 @@ concurrency:
 
 jobs:
   setup:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
-        lmod7: Lmod-7.8.22
-        lmod8: Lmod-8.7.6
-        modulesTcl: modules-tcl-1.147
-        modules3: modules-3.2.10
-        modules4: modules-4.1.4
+        lmod8: Lmod-8.7.58
+        modules4: modules-4.5.3
+        modules5: modules-5.3.1
     steps:
       - run: "true"
   build:
     needs: setup
-    runs-on: ubuntu-20.04
+    runs-on: ${{matrix.os || 'ubuntu-24.04'}}
     strategy:
       matrix:
-        python: [3.6]
+        # Python 3.10 is default in Ubuntu 22.04
+        python: ['3.10']
         modules_tool:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
-          - ${{needs.setup.outputs.lmod7}}
           - ${{needs.setup.outputs.lmod8}}
-          - ${{needs.setup.outputs.modulesTcl}}
-          - ${{needs.setup.outputs.modules3}}
           - ${{needs.setup.outputs.modules4}}
-        lc_all: [""]
+          - ${{needs.setup.outputs.modules5}}
         include:
-          # Test different Python 3 versions with Lmod 8.x
-          - python: 3.7
+          # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
+          - python: '3.7'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-          - python: 3.8
+            os: ubuntu-22.04
+          - python: '3.8'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-          - python: 3.9
-            modules_tool: ${{needs.setup.outputs.lmod8}}
-          - python: '3.10'
+          - python: '3.9'
             modules_tool: ${{needs.setup.outputs.lmod8}}
           - python: '3.11'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-          # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
-          # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
-          - python: 3.6
+          - python: '3.12'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+          - python: '3.13'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            lc_all: C
       fail-fast: false
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
 
     - name: install OS & Python packages
       run: |
         # for modules tool
-        APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+        APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
         # for GitPython, python-hglib
         APT_PKGS+=" git mercurial"
         # dep for GC3Pie
@@ -79,19 +73,18 @@ jobs:
           sudo apt-get install $APT_PKGS
         fi
 
-        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
-        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
-            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
-        fi
         # Python packages
         pip --version
         pip install --upgrade pip
         pip --version
         pip install -r requirements.txt
+        if ! python -c "import distutils" 2> /dev/null; then
+            # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+            pip install --upgrade setuptools
+        fi
         # git config is required to make actual git commits (cfr. tests for GitRepository)
-        git config --global user.name "Travis CI"
-        git config --global user.email "[email protected]"
+        git config --global user.name "Github Actions"
+        git config --global user.email "[email protected]"
         git config --get-regexp 'user.*'
 
     - name: install GitHub token (if available)
@@ -101,12 +94,12 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
+        if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.9 ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
           fi
           echo "GitHub token installed!"
         else
@@ -129,7 +122,7 @@ jobs:
       run: |
           # make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements,
           # since EasyBuild should not have a runtime requirement on setuptools
-          SETUPTOOLS_IMPORTS=$(egrep -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true)
+          SETUPTOOLS_IMPORTS=$(egrep --exclude setup.py -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true)
           test "x$SETUPTOOLS_IMPORTS" = "x" || (echo "Found setuptools and/or pkg_resources imports in easybuild/:\n${SETUPTOOLS_IMPORTS}" && exit 1)
 
     - name: install sources
@@ -143,26 +136,24 @@ jobs:
     - name: run test suite
       env:
         EB_VERBOSE: 1
-        LC_ALL: ${{matrix.lc_all}}
+        LC_ALL: "" 
       run: |
           # run tests *outside* of checked out easybuild-framework directory,
           # to ensure we're testing installed version (see previous step)
           cd $HOME
           # initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
-          source $(cat $HOME/mod_init); type module
+          source $(cat $HOME/mod_init)
+          type module
+          module --version
           # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
           # also pick up changes to $PATH set by sourcing $MOD_INIT
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           export PATH=$PREFIX/bin:$(cat $HOME/path)
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod
@@ -191,7 +182,16 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS="no GitHub token available"
+            IGNORE_PATTERNS+="|skipping SvnRepository test"
+            IGNORE_PATTERNS+="|requires Lmod as modules tool"
+            IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.7"
+            IGNORE_PATTERNS+="|from cryptography.* import "
+            IGNORE_PATTERNS+="|Blowfish"
+            IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+            IGNORE_PATTERNS+="|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)