ubuntu-ci-x86_64-oneapi.yaml

name: ubuntu-ci-c6a-x86_64-oneapi-build
on:
  pull_request:
    paths-ignore:
      - 'configs/sites/**'
      - '!configs/sites/tier2/*.default/*'
      - '**.md'
      - '.github/CODEOWNERS'
      - '.github/pull_request_template.md'
      - '.github/ISSUE_TEMPLATE/*'
      - '.gitignore'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  # DH* REVERT ME AFTER MERGE
  BUILD_CACHE_PATH: /home/ubuntu/spack-stack/build-cache-new-spack-v1
  SOURCE_CACHE_PATH: /home/ubuntu/spack-stack/source-cache

jobs:
  ubuntu-ci-c6a-x86_64-oneapi-build:
    runs-on: [ubuntu-ci-c6a-x86_64]

    steps:
      - name: cleanup
        run: |
          pwd
          ls -lart
          set +e
          find ./* -type d -exec chmod u+xw {} \;
          set -e
          rm -fr *

      - name: checkout
        uses: actions/checkout@v4
        with:
          submodules: true

      - name: prepare-directories
        run: |
          mkdir -p ${BUILD_CACHE_PATH}
          mkdir -p ${SOURCE_CACHE_PATH}

      - name: create-buildcache
        run: |
          # Set up spack-stack
          source ./setup.sh

          declare -a TEMPLATES=("unified-dev" "skylab-dev" "neptune-dev")
          for TEMPLATE in "${TEMPLATES[@]}"; do
            if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then
              export ENVNAME=ue-oneapi-2024.2.0-buildcache
            elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then
              export ENVNAME=se-oneapi-2024.2.0-buildcache
            elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then
              export ENVNAME=ce-oneapi-2024.2.0-buildcache
            elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
              export ENVNAME=ne-oneapi-2024.2.0-buildcache
            fi
            echo "Creating environment ${ENVNAME} from template ${TEMPLATE}"

            export ENVDIR=$PWD/envs/${ENVNAME}
            spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler oneapi
            spack env activate ${ENVDIR}

            # Turn off variants "adp" and "jedi" for neptune-dev (NRL internal only)
            if [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
              sed -i 's/+adp/~adp/g' ${ENVDIR}/spack.yaml
              sed -i 's/+jedi/~jedi/g' ${ENVDIR}/spack.yaml
            fi

            unset SPACK_DISABLE_LOCAL_CONFIG
            export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"

            # Find external packages
            spack external find --scope system \
                --exclude bison --exclude openssl \
                --exclude curl --exclude python \
                --exclude meson --exclude gmake
            spack external find --scope system grep
            spack external find --scope system sed
            spack external find --scope system perl
            spack external find --scope system wget
            spack external find --scope system texlive
            spack external find --scope system mysql

            # Find compilers
            spack compiler find --scope system

            # For Intel oneAPI
            source /etc/profile.d/modules.sh
            module use /opt/intel/oneapi/modulefiles

            cat .github/workflows/ubuntu-ci-x86_64-oneapi.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml

            export SPACK_DISABLE_LOCAL_CONFIG=true
            unset SPACK_SYSTEM_CONFIG_PATH

            # For buildcaches
            spack config add config:install_tree:padded_length:200

            # Set compiler and MPI specs
            spack config add "packages:mpi:buildable:False"
            spack config add "packages:mpi:require:['intel-oneapi-mpi@2021.13']"

            # Add additional variants for MET packages, different from config/common/packages.yaml
            spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"

            ## Pin gmake to avoid duplicate packages (excluded gmake from spack external find above)
            #spack config add "packages:gmake:require:'@:4.2'"

            ## Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI
            #sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/`

            # Don't generate ecflow module when using external package
            spack config add "modules:default:tcl:exclude:[ecflow]"

            # Concretize, check for duplicates and for correct compilers being used
            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}
            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms -i ip
            spack stack check-preferred-compiler 2>&1 | tee log.check-preferred-compiler.${ENVNAME}

            # Add and update source cache
            spack mirror add local-source file://${SOURCE_CACHE_PATH}/
            spack mirror create -a -d ${SOURCE_CACHE_PATH}/

            # Add binary cache
            set +e
            spack mirror add local-binary file://${BUILD_CACHE_PATH}/
            spack buildcache update-index local-binary
            set -e
            echo "Packages in spack binary cache:"
            spack buildcache list

            # Break installation up in pieces and create build caches in between
            # This allows us to "spin up" builds that altogether take longer than
            # six hours, and/or fail later in the build process.

            if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then
              # base-env
              echo "base-env ..."
              spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env
              spack buildcache create -u local-binary base-env
              # jedi-base-env
              echo "jedi-base-env ..."
              spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.${ENVNAME}.jedi-base-env
              spack buildcache create -u local-binary jedi-base-env
              # jedi-ufs-env
              echo "jedi-ufs-env ..."
              spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.${ENVNAME}.jedi-ufs-env
              spack buildcache create -u local-binary jedi-ufs-env
            elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
              # base-env
              echo "base-env ..."
              spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env
              spack buildcache create -u local-binary base-env
              # neptune-python-env
              echo "neptune-env ..."
              spack install --fail-fast --source --no-check-signature neptune-python-env 2>&1 | tee log.install.${ENVNAME}.neptune-env
              spack buildcache create -u local-binary neptune-python-env
            fi

            # the rest
            echo "${TEMPLATE} ..."
            spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all
            spack buildcache create -u local-binary

            # Check for libirc.so in shared libraries (returns 1=error if found)
            ${SPACK_STACK_DIR}/util/check_libirc.sh

            # Remove buildcache config settings
            spack config remove config:install_tree:padded_length

            # Next steps: synchronize source and build cache to a central/combined mirror?

            # Cleanup
            spack clean -a
            spack env deactivate

          done

      - name: create-env
        run: |
          # Set up spack-stack
          source ./setup.sh
          export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache
          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
          export ENVNAME=ue-oneapi-2024.2.0
          export ENVDIR=$PWD/envs/${ENVNAME}
          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
          spack env activate ${ENVDIR}

          # Concretize, check for duplicates and for correct compilers being used
          spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}
          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms -i ip
          spack stack check-preferred-compiler 2>&1 | tee log.check-preferred-compiler.${ENVNAME}

          # Add binary cache back in
          spack mirror add local-binary file://${BUILD_CACHE_PATH}/
          echo "Packages in combined spack build caches:"
          spack buildcache list

          # Install from cache
          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all

          # For spack-built Intel oneAPI runtime to find libintlc.so.5 etc.
          export LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/2024.2/lib:$LD_LIBRARY_PATH"

          # Check shared libraries
          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check

          # Check for libirc.so in shared libraries (returns 1=error if found)
          ${SPACK_STACK_DIR}/util/check_libirc.sh

          # Create modules
          spack clean -a
          spack module tcl refresh -y
          spack stack setup-meta-modules
          spack env deactivate

      - name: test-env
        run: |
          source /etc/profile.d/modules.sh
          module use /opt/intel/oneapi/modulefiles
          module use /home/ubuntu/spack-stack/modulefiles

          export ENVNAME=ue-oneapi-2024.2.0
          export ENVDIR=$PWD/envs/${ENVNAME}
          ls -l ${ENVDIR}/modules/Core

          module use ${ENVDIR}/modules/Core
          module load stack-intel-oneapi-compilers/2024.2.0
          module load stack-intel-oneapi-mpi/2021.13
          module available

          module load jedi-ufs-env
          module load ewok-env
          module load soca-env
          module list