|
| 1 | +# Hooks for HPC2N site changes. |
| 2 | +# |
| 3 | +# Author: Ake Sandgren, HPC2N |
| 4 | + |
| 5 | +import os |
| 6 | + |
| 7 | +from distutils.version import LooseVersion |
| 8 | +from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS |
| 9 | +from easybuild.tools.filetools import apply_regex_substitutions |
| 10 | +from easybuild.tools.build_log import EasyBuildError |
| 11 | +from easybuild.tools.modules import get_software_root |
| 12 | +from easybuild.tools.systemtools import get_shared_lib_ext |
| 13 | + |
| 14 | +# Add/remove dependencies and/or patches |
| 15 | +# Access to the raw values before templating and such. |
| 16 | +def parse_hook(ec, *args, **kwargs): |
| 17 | + |
| 18 | + # Internal helper function |
| 19 | + def add_extra_dependencies(ec, dep_type, extra_deps): |
| 20 | + """dep_type: must be in DEPENDENCY_PARAMETERS or 'osdependencies'""" |
| 21 | + ec.log.info("[parse hook] Adding %s: %s" % (dep_type, extra_deps)) |
| 22 | + |
| 23 | + if dep_type in DEPENDENCY_PARAMETERS: |
| 24 | + for dep in extra_deps: |
| 25 | + ec[dep_type].append(dep) |
| 26 | + elif dep_type == 'osdependencies': |
| 27 | + if isinstance(extra_deps, tuple): |
| 28 | + ec[dep_type].append(extra_deps) |
| 29 | + else: |
| 30 | + raise EasyBuildError("parse_hook: Type of extra_deps argument (%s), for 'osdependencies' must be " |
| 31 | + "tuple, found %s" % (extra_deps, type(extra_deps))) |
| 32 | + else: |
| 33 | + raise EasyBuildError("parse_hook: Incorrect dependency type in add_extra_dependencies: %s" % dep_type) |
| 34 | + |
| 35 | + extra_deps = [] |
| 36 | + |
| 37 | + if ec.name == 'OpenMPI': |
| 38 | + if LooseVersion(ec.version) >= LooseVersion('2') and LooseVersion(ec.version) < LooseVersion('2.1.2'): |
| 39 | + ec.log.info("[parse hook] Adding pmi and lustre patches") |
| 40 | + if LooseVersion(ec.version) < LooseVersion('2.1.1'): |
| 41 | + ec['patches'].append('OpenMPI-2.0.0_fix_bad-include_of_pmi_h.patch') |
| 42 | + |
| 43 | + if LooseVersion(ec.version) < LooseVersion('2.0.2'): |
| 44 | + ec['patches'].append('OpenMPI-2.0.1_fix_lustre.patch') |
| 45 | + elif LooseVersion(ec.version) < LooseVersion('2.1'): |
| 46 | + ec['patches'].append('OpenMPI-2.0.2_fix_lustre.patch') |
| 47 | + elif LooseVersion(ec.version) < LooseVersion('2.1.1'): |
| 48 | + ec['patches'].append('OpenMPI-2.1.0_fix_lustre.patch') |
| 49 | + else: |
| 50 | + ec['patches'].append('OpenMPI-2.1.1_fix_lustre.patch') |
| 51 | + |
| 52 | + if LooseVersion(ec.version) == LooseVersion('4.0.0'): |
| 53 | + ec['patches'].append('OpenMPI-4.0.0_fix_configure_bug.patch') |
| 54 | + |
| 55 | + if LooseVersion(ec.version) >= LooseVersion('2.1'): |
| 56 | + pmix_version = '1.2.5' |
| 57 | + ucx_version = '1.4.0' |
| 58 | + if LooseVersion(ec.version) >= LooseVersion('3'): |
| 59 | + pmix_version = '2.2.1' |
| 60 | + if LooseVersion(ec.version) >= LooseVersion('4'): |
| 61 | + pmix_version = '3.0.2' # OpenMPI 4.0.0 is not compatible with PMIx 3.1.x |
| 62 | + |
| 63 | + extra_deps.append(('PMIx', pmix_version)) |
| 64 | + # Use of external PMIx requires external libevent |
| 65 | + # But PMIx already has it as a dependency so we don't need |
| 66 | + # to explicitly set it. |
| 67 | + |
| 68 | + extra_deps.append(('UCX', ucx_version)) |
| 69 | + |
| 70 | + if ec.name == 'impi': |
| 71 | + pmix_version = '3.1.1' |
| 72 | + extra_deps.append(('PMIx', pmix_version)) |
| 73 | + |
| 74 | + if extra_deps: |
| 75 | + add_extra_dependencies(ec, 'dependencies', extra_deps) |
| 76 | + |
| 77 | + |
| 78 | +def pre_configure_hook(self, *args, **kwargs): |
| 79 | + if self.name == 'GROMACS': |
| 80 | + # HPC2N always uses -DGMX_USE_NVML=ON on GPU builds |
| 81 | + if get_software_root('CUDA'): |
| 82 | + self.log.info("[pre-configure hook] Adding -DGMX_USE_NVML=ON") |
| 83 | + self.cfg.update('configopts', "-DGMX_USE_NVML=ON ") |
| 84 | + |
| 85 | + if self.name == 'OpenMPI': |
| 86 | + extra_opts = "" |
| 87 | + # Old versions don't work with PMIx, use slurms PMI1 |
| 88 | + if LooseVersion(self.version) < LooseVersion('2.1'): |
| 89 | + extra_opts += "--with-pmi=/lap/slurm " |
| 90 | + if LooseVersion(self.version) >= LooseVersion('2'): |
| 91 | + extra_opts += "--with-munge " |
| 92 | + |
| 93 | + # Using PMIx dependency in easyconfig, see above |
| 94 | + if LooseVersion(self.version) >= LooseVersion('2.1'): |
| 95 | + if get_software_root('PMIx'): |
| 96 | + extra_opts += "--with-pmix=$EBROOTPMIX " |
| 97 | + # Use of external PMIx requires external libevent |
| 98 | + # We're using the libevent that comes from the PMIx dependency |
| 99 | + if get_software_root('libevent'): |
| 100 | + extra_opts += "--with-libevent=$EBROOTLIBEVENT " |
| 101 | + else: |
| 102 | + raise EasyBuildError("Error in pre_configure_hook for OpenMPI: External use of PMIx requires " |
| 103 | + "external libevent, which was not found. " |
| 104 | + "Check parse_hook for dependency settings.") |
| 105 | + else: |
| 106 | + raise EasyBuildError("Error in pre_configure_hook for OpenMPI: PMIx not defined in dependencies. " |
| 107 | + "Check parse_hook for dependency settings.") |
| 108 | + |
| 109 | + if get_software_root('UCX'): |
| 110 | + extra_opts += "--with-ucx=$EBROOTUCX " |
| 111 | + |
| 112 | + if LooseVersion(self.version) >= LooseVersion('2'): |
| 113 | + extra_opts += "--with-cma " |
| 114 | + extra_opts += "--with-lustre " |
| 115 | + |
| 116 | + # We still need to fix the knem package to install its |
| 117 | + # pkg-config .pc file correctly, and we need a more generic |
| 118 | + # install dir. |
| 119 | + # extra_opts += "--with-knem=/opt/knem-1.1.2.90mlnx1 " |
| 120 | + |
| 121 | + self.log.info("[pre-configure hook] Adding %s" % extra_opts) |
| 122 | + self.cfg.update('configopts', extra_opts) |
| 123 | + |
| 124 | + if LooseVersion(self.version) >= LooseVersion('2.1'): |
| 125 | + self.log.info("[pre-configure hook] Re-enabling ucx") |
| 126 | + self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ') |
| 127 | + |
| 128 | + self.log.info("[pre-configure hook] Re-enabling dlopen") |
| 129 | + self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') |
| 130 | + |
| 131 | + if self.name == 'PMIx': |
| 132 | + self.log.info("[pre-configure hook] Adding --with-munge") |
| 133 | + self.cfg.update('configopts', "--with-munge ") |
| 134 | + if LooseVersion(self.version) >= LooseVersion('2'): |
| 135 | + self.log.info("[pre-configure hook] Adding --with-tests-examples") |
| 136 | + self.cfg.update('configopts', "--with-tests-examples ") |
| 137 | + self.log.info("[pre-configure hook] Adding --disable-per-user-config-files") |
| 138 | + self.cfg.update('configopts', "--disable-per-user-config-files") |
| 139 | + |
| 140 | + |
| 141 | +def pre_build_hook(self, *args, **kwargs): |
| 142 | + if self.name == 'pyslurm': |
| 143 | + self.log.info("[pre-build hook] Adding --slurm=/lap/slurm") |
| 144 | + self.cfg.update('buildopts', "--slurm=/lap/slurm ") |
| 145 | + |
| 146 | + |
| 147 | +def post_install_hook(self, *args, **kwargs): |
| 148 | + if self.name == 'impi': |
| 149 | + # Fix mpirun from IntelMPI to explicitly unset I_MPI_PMI_LIBRARY |
| 150 | + # it can only be used with srun. |
| 151 | + self.log.info("[post-install hook] Unset I_MPI_PMI_LIBRARY in mpirun") |
| 152 | + apply_regex_substitutions(os.path.join(self.installdir, "intel64", "bin", "mpirun"), [ |
| 153 | + (r'^(#!/bin/sh.*)$', r'\1\nunset I_MPI_PMI_LIBRARY'), |
| 154 | + ]) |
| 155 | + |
| 156 | + |
| 157 | +def pre_module_hook(self, *args, **kwargs): |
| 158 | + if self.name == 'impi': |
| 159 | + # Add I_MPI_PMI_LIBRARY to module for IntelMPI so it works with |
| 160 | + # srun. |
| 161 | + self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module") |
| 162 | + # Must be done this way, updating self.cfg['modextravars'] |
| 163 | + # directly doesn't work due to templating. |
| 164 | + en_templ = self.cfg.enable_templating |
| 165 | + self.cfg.enable_templating = False |
| 166 | + shlib_ext = get_shared_lib_ext() |
| 167 | + pmix_root = get_software_root('PMIx') |
| 168 | + if pmix_root: |
| 169 | + mpi_type = 'pmix_v3' |
| 170 | + self.cfg['modextravars'].update({ |
| 171 | + 'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext) |
| 172 | + }) |
| 173 | + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) |
| 174 | + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. |
| 175 | + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) |
| 176 | + else: |
| 177 | + self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) |
| 178 | + self.cfg.enable_templating = en_templ |
| 179 | + |
| 180 | + if self.name == 'OpenBLAS': |
| 181 | + self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module") |
| 182 | + self.cfg.update('modluafooter', 'if ((mode() == "load" and os.getenv("OMP_NUM_THREADS") == nil) ' |
| 183 | + 'or (mode() == "unload" and os.getenv("__OpenBLAS_set_OMP_NUM_THREADS") == "1")) then ' |
| 184 | + 'setenv("OMP_NUM_THREADS","1"); setenv("__OpenBLAS_set_OMP_NUM_THREADS", "1") end') |
| 185 | + |
| 186 | + if self.name == 'OpenMPI': |
| 187 | + if LooseVersion(self.version) < LooseVersion('2.1'): |
| 188 | + mpi_type = 'openmpi' |
| 189 | + elif LooseVersion(self.version) < LooseVersion('3'): |
| 190 | + mpi_type = 'pmix_v1' |
| 191 | + elif LooseVersion(self.version) < LooseVersion('4'): |
| 192 | + mpi_type = 'pmix_v2' |
| 193 | + else: |
| 194 | + mpi_type = 'pmix_v3' |
| 195 | + |
| 196 | + self.log.info("[pre-module hook] Set SLURM_MPI_TYPE=%s in OpenMPI module" % mpi_type) |
| 197 | + # Must be done this way, updating self.cfg['modextravars'] |
| 198 | + # directly doesn't work due to templating. |
| 199 | + en_templ = self.cfg.enable_templating |
| 200 | + self.cfg.enable_templating = False |
| 201 | + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) |
| 202 | + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. |
| 203 | + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) |
| 204 | + self.cfg.enable_templating = en_templ |
| 205 | + |
| 206 | + if self.name == 'PMIx': |
| 207 | + # This is a, hopefully, temporary workaround for https://github.com/pmix/pmix/issues/1114 |
| 208 | + if LooseVersion(self.version) > LooseVersion('2') and LooseVersion(self.version) < LooseVersion('3'): |
| 209 | + self.log.info("[pre-module hook] Set PMIX_MCA_gds=^ds21 in PMIx module") |
| 210 | + en_templ = self.cfg.enable_templating |
| 211 | + self.cfg.enable_templating = False |
| 212 | + self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'}) |
| 213 | + self.cfg.enable_templating = en_templ |
0 commit comments