Merge branch 'easybuilders:develop' into allow_try_for_multi_tc

Author: Flamefire

.github/workflows/container_tests.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [2.7, 3.7]
+        python: [3.7]
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/container_tests_apptainer.yml

@@ -14,14 +14,14 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [2.7, 3.7]
+        python: [3.7]
         apptainer: [1.0.0, 1.1.7]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/eb_command.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/end2end.yml

@@ -0,0 +1,61 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+  build_publish:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        container:
+          - centos-7.9
+          - centos-8.5
+          - fedora-36
+          - opensuse-15.4
+          - rockylinux-8.8
+          - rockylinux-9.2
+          - ubuntu-20.04
+          - ubuntu-22.04
+      fail-fast: false
+    container:
+      image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: download and unpack easyblocks and easyconfigs repositories
+        run: |
+          cd $HOME
+          for pkg in easyblocks easyconfigs; do
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
+              tar xfz develop.tar.gz
+              rm -f develop.tar.gz
+          done
+
+      - name: Set up environment
+        shell: bash
+        run: |
+          # collect environment variables to be set in subsequent steps in script that can be sourced
+          echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+
+      - name: Run commands to check test environment
+        shell: bash
+        run: |
+          cmds=(
+            "whoami"
+            "pwd"
+            "env | sort"
+            "eb --version"
+            "eb --show-system-info"
+            "eb --check-eb-deps"
+            "eb --show-config"
+          )
+          for cmd in "${cmds[@]}"; do
+              echo ">>> $cmd"
+              sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+          done
+
+      - name: End-to-end test of installing bzip2 with EasyBuild
+        shell: bash
+        run: |
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"

.github/workflows/linting.yml

@@ -13,13 +13,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/unit_tests.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.6]
+        python: [3.6]
         modules_tool:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
@@ -34,54 +34,32 @@ jobs:
           - ${{needs.setup.outputs.modulesTcl}}
           - ${{needs.setup.outputs.modules3}}
           - ${{needs.setup.outputs.modules4}}
-        module_syntax: [Lua, Tcl]
         lc_all: [""]
-        # don't test with Lua module syntax (only supported in Lmod)
-        exclude:
-          - modules_tool: ${{needs.setup.outputs.modulesTcl}}
-            module_syntax: Lua
-          - modules_tool: ${{needs.setup.outputs.modules3}}
-            module_syntax: Lua
-          - modules_tool: ${{needs.setup.outputs.modules4}}
-            module_syntax: Lua
         include:
-          # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
+          # Test different Python 3 versions with Lmod 8.x
           - python: 3.5
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: 3.7
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: 3.8
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
-          - python: 3.8
-            modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Tcl
           - python: 3.9
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: '3.10'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
-          - python: '3.11'
-            modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: '3.11'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Tcl
           # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
           # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
           - python: 3.6
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
             lc_all: C
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -125,16 +103,11 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
-        # and only when testing with Lua as module syntax,
-        # to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then
+        if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            if [ "x${{matrix.python}}" == 'x2.6' ];
-              then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
-              else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            fi;
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
             python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
           fi
           echo "GitHub token installed!"
@@ -172,8 +145,6 @@ jobs:
     - name: run test suite
       env:
         EB_VERBOSE: 1
-        EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
-        TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
         LC_ALL: ${{matrix.lc_all}}
       run: |
           # run tests *outside* of checked out easybuild-framework directory,
@@ -198,23 +169,32 @@ jobs:
           else
             export EASYBUILD_MODULES_TOOL=Lmod
           fi
-          export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL
-          eb --show-config
-          # gather some useful info on test system
-          eb --show-system-info
-          # check GitHub configuration
-          eb --check-github --github-user=easybuild_test
-          # create file owned by root but writable by anyone (used by test_copy_file)
-          sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-          sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-          # silence deprecation warning when using Python 2, since it breaks a bunch of tests
-          if [ "${{matrix.python}}" == '2.7' ]; then
-            export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2
-          fi
-          # run test suite
-          python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
-          # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
-          # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
-          PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
-          test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
+          export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL}
+
+          # Run tests with LUA and Tcl module syntax (where supported)
+          for module_syntax in Lua Tcl; do
+            # Only Lmod supports Lua
+            if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then
+              echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
+              continue
+            fi
+            printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
+            export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
+            export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
+
+            eb --show-config
+            # gather some useful info on test system
+            eb --show-system-info
+            # check GitHub configuration
+            eb --check-github --github-user=easybuild_test
+            # create file owned by root but writable by anyone (used by test_copy_file)
+            sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+            sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+            # run test suite
+            python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
+            # try and make sure output of running tests is clean (no printed messages/warnings)
+            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
+            PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
+            test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
+          done

.github/workflows/unit_tests_python2.yml

@@ -0,0 +1,81 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: EasyBuild framework unit tests (python2)
+on: [push, pull_request]
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+concurrency:
+  group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
+  cancel-in-progress: true
+
+jobs:
+  test_python2:
+    runs-on: ubuntu-20.04
+    container:
+      # CentOS 7.9 container that already includes Lmod & co,
+      # see https://github.com/easybuilders/easybuild-containers
+      image: ghcr.io/easybuilders/centos-7.9-amd64
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: install Python packages
+      run: |
+        # Python packages
+        python2 -V
+        python2 -m pip --version
+        python2 -m pip install --upgrade pip
+        python2 -m pip --version
+        # strip out GC3Pie since installation with ancient setuptools (0.9.8) fails
+        sed -i '/GC3Pie/d' requirements.txt
+        python2 -m pip install -r requirements.txt
+        # git config is required to make actual git commits (cfr. tests for GitRepository)
+        sudo -u easybuild git config --global user.name "GitHub Actions"
+        sudo -u easybuild git config --global user.email "[email protected]"
+        sudo -u easybuild git config --get-regexp 'user.*'
+
+    - name: install GitHub token (if available)
+      env:
+        # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
+        # this token is not available in pull requests, so tests that require it are skipped in PRs,
+        # and are only run after the PR gets merged
+        GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
+      run: |
+        # tests that require a GitHub token are skipped automatically when no GitHub token is available
+        if [ ! -z $GITHUB_TOKEN ]; then
+          sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+          echo "GitHub token installed!"
+        else
+          echo "Installation of GitHub token skipped!"
+        fi
+
+    - name: install sources
+      run: |
+          # install from source distribution tarball, to test release as published on PyPI
+          python2 setup.py sdist
+          ls dist
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+
+    - name: run test suite
+      run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that)
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH"
+          ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2"
+          # run EasyBuild command via (non-root) easybuild user + login shell
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version"
+          # show active EasyBuild configuration
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config"
+          # gather some useful info on test system
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info"
+          # check GitHub configuration
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test"
+          # create file owned by root but writable by anyone (used by test_copy_file)
+          sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+          sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+          # run test suite (via easybuild user + login shell)
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite"