test-software.eessi.io.yml

# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Check for missing software installations in software.eessi.io
on:
  push:
    branches: [ "*-software.eessi.io" ]
  pull_request:
  workflow_dispatch:
permissions:
  contents: read # to fetch code (actions/checkout)
env:
  EESSI_VERSION: '2023.06'
  EESSI_ACCELERATOR_TARGETS: |
    x86_64/amd/zen2:
      - nvidia/cc80
    x86_64/amd/zen3:
      - nvidia/cc80
jobs:
  check_missing:
    strategy:
      fail-fast: false
      matrix:
        include:
          # Arm CPU targets
          - runs_on: ubuntu-24.04-arm
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic
          - runs_on: ubuntu-24.04-arm
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_n1
          - runs_on: ubuntu-24.04-arm
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_v1
          - runs_on: ubuntu-24.04-arm
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/nvidia/grace
          # x86_64 CPU targets
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen2
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen3
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen4
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/haswell
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/sapphirerapids
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/skylake_avx512
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/icelake
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/cascadelake
          - runs_on: ubuntu-24.04
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
    runs-on: ${{ matrix.runs_on }}
    steps:
        - name: Check out software-layer repository
          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
          with:
              fetch-depth: 0  # Fetch all history for all branches and tags

        - name: Show host system info
          run: |
            echo "/proc/cpuinfo:"
            cat /proc/cpuinfo
            echo
            echo "lscpu:"
            lscpu

        - name: Mount EESSI CernVM-FS pilot repository
          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
          with:
              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
              cvmfs_http_proxy: DIRECT
              cvmfs_repositories: software.eessi.io

        - name: Test check_missing_installations.sh script
          run: |
              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
              # to prevent issues with checks in the Easybuild configuration that use this variable
              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION}
              export EESSI_OS_TYPE=linux
              env | grep ^EESSI | sort

              # first check the CPU-only builds for this CPU target
              echo "first run check_missing_installations.sh for CPU-only builds"
              for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} .github/workflows/scripts/only_latest_easystacks.sh); do
                  eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
                  echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
                  module purge
                  module load EasyBuild/${eb_version}
                  which eb
                  eb --version
                  ./check_missing_installations.sh ${easystack_file}
                  ec=$?
                  if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
              done

              # now check the accelerator builds for this CPU target
              accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]")
              if [ -z ${accelerators} ]; then
                  echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
              else
                  for accel in ${accelerators}; do
                      module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
                      echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
                      for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
                          eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
                          echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
                          module purge
                          module load EasyBuild/${eb_version}
                          which eb
                          eb --version
                          ./check_missing_installations.sh ${easystack_file}
                          ec=$?
                          if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
                      done
                      module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
                  done
              fi

        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
          run: |
              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
              # to prevent issues with checks in the Easybuild configuration that use this variable
              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
              module load EasyBuild
              which eb
              eb --version
              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION}
              export EESSI_OS_TYPE=linux
              env | grep ^EESSI | sort
              # create dummy easystack file with a single entry (something that is not installed in EESSI)
              easystack_file="test.yml"
              echo "easyconfigs:" > ${easystack_file}
              echo "  - GCC-8.3.0:" >> ${easystack_file}
              echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
              cat ${easystack_file}
              # note, check_missing_installations.sh exits 1 if a package was
              # missing, which is intepreted as false (exit code based, not
              # boolean logic), hence when the script exits 0 if no package was
              # missing it is interpreted as true, thus the test did not capture
              # the missing package
              if ./check_missing_installations.sh ${easystack_file}; then
                echo "did NOT capture missing package; test FAILED"
                exit 1
              else
                echo "captured missing package; test PASSED"
                exit 0
              fi

        - name: Check that EasyBuild hook is up to date
          if: ${{ github.event_name == 'pull_request' }}
          run: |
              FILE="eb_hooks.py"
              TEMP_FILE="$(mktemp)"

              # Fetch base branch
              git fetch origin ${{ github.base_ref }}

              # Check if the hooks has changed in the PR
              if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then
                echo "Hooks changed in PR. Using PR version."
                cp "$FILE" "$TEMP_FILE"
              else
                echo "File not changed in PR. Using default branch version."
                git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
              fi

              # Compare the hooks to what is shipped in the repository
              # (it is overkill, but harmless, to do this for every architecture)
              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
              module load EESSI-extend
              diff "$TEMP_FILE" "$EASYBUILD_HOOKS"