Limit accelerator installations to CPU only

Author: ocaisa

EESSI-extend-easybuild.eb

@@ -104,6 +104,16 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then
     LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
   end
   easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH")
+  eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET")
+  if (eessi_accelerator_target ~= nil) then
+    cuda_compute_capability = string.match(eessi_accelerator_target, "^nvidia/cc([0-9][0-9])$")
+    if (cuda_compute_capability ~= nil) then
+      easybuild_installpath = pathJoin(easybuild_installpath, 'accel', eessi_accelerator_target)
+      easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) ..  "." .. cuda_compute_capability:sub(2, 2)
+    else
+      LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target)
+    end
+  end
 else
   -- Deal with user and project installs
   project_install = os.getenv("EESSI_PROJECT_INSTALL")

eb_hooks.py

@@ -421,6 +421,37 @@ def pre_fetch_hook(self, *args, **kwargs):
     if cpu_target == CPU_TARGET_ZEN4:
         pre_fetch_hook_zen4_gcccore1220(self, *args, **kwargs)
 
+    # Always check the software installation path
+    pre_fetch_hook_check_installation_path(self, *args, **kwargs)
+
+
+# Check the installation path so we verify that accelerator software always gets installed into the correct location
+def pre_fetch_hook_check_installation_path(self, *args, **kwargs):
+    # When we know the CUDA status, we will need to verify the installation path
+    # if we are doing an EESSI or host_injections installation
+    accelerator_deps = ['CUDA']
+    strict_eessi_installation = (
+        bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) or
+        self.installdir.startswith(HOST_INJECTIONS_LOCATION))
+    if strict_eessi_installation:
+        dependency_names = self.cfg.dependency_names()
+        if self.cfg.name in accelerator_deps or any(dep in dependency_names for dep in accelerator_deps):
+            # Make sure the path is an accelerator location
+            if "/accel/" not in self.installdir:
+                raise EasyBuildError(
+                    f"It seems you are trying to install an accelerator package {self.cfg.name} into a "
+                    f"non-accelerator location {self.installdir}. You need to reconfigure your installation to target "
+                    "the correct location."
+                    )
+        else:
+            # If we don't have an accelerator dependency then we should be in a CPU installation path
+            if "/accel/" in self.installdir:
+                raise EasyBuildError(
+                    f"It seems you are trying to install a CPU-only package {self.cfg.name} into accelerator location "
+                    f"{self.installdir}. If this is a dependency of the package you are really interested in you will "
+                    "need to first install the CPU-only dependencies of that package."
+                    )
+
 
 def pre_fetch_hook_zen4_gcccore1220(self, *args, **kwargs):
     """Use --force --module-only if building a foss-2022b-based EasyConfig for Zen4.

init/bash

@@ -32,6 +32,8 @@ if [ $? -eq 0 ]; then
     if [ ! -z ${EESSI_MODULEPATH_ACCEL} ]; then
         show_msg "Prepending $EESSI_MODULEPATH_ACCEL to \$MODULEPATH..."
         module use $EESSI_MODULEPATH_ACCEL
+        show_msg "Prepending $EESSI_SITE_MODULEPATH_ACCEL to \$MODULEPATH..."
+        module use $EESSI_SITE_MODULEPATH_ACCEL
     fi
 
     #show_msg ""

init/eessi_environment_variables

@@ -112,6 +112,8 @@ if [ -d $EESSI_PREFIX ]; then
         elif [ -d $EESSI_SOFTWARE_PATH ]; then
           export EESSI_SITE_SOFTWARE_PATH=${EESSI_SOFTWARE_PATH/versions/host_injections}
           show_msg "Using ${EESSI_SITE_SOFTWARE_PATH} as the site extension directory for installations."
+          export EESSI_SITE_ACCEL_SOFTWARE_PATH=${EESSI_ACCEL_SOFTWARE_PATH/versions/host_injections}
+          show_msg "Using ${EESSI_SITE_ACCEL_SOFTWARE_PATH} as the site extension directory for accelerated installations."
           # Allow for use of alternative module tree shipped with EESSI
           if [ -z ${EESSI_MODULE_SUBDIR+x} ]; then
             # EESSI_MODULE_SUBDIR not set
@@ -140,8 +142,11 @@ if [ -d $EESSI_PREFIX ]; then
           if [ -d ${EESSI_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR} ]; then
               export EESSI_MODULEPATH_ACCEL=${EESSI_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR}
               show_msg "Using ${EESSI_MODULEPATH_ACCEL} as additional directory (for accelerators) to be added to MODULEPATH."
+              export EESSI_SITE_MODULEPATH_ACCEL=${EESSI_SITE_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR}
+              show_msg "Using ${EESSI_SITE_MODULEPATH_ACCEL} as additional site extension directory (for accelerators) to be added to MODULEPATH."
           fi
 
+
           # Fix wrong path for RHEL >=8 libcurl 
           # This is required here because we ship curl in our compat layer. If we only provided
           # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild

init/modules/EESSI/2023.06.lua

@@ -144,6 +144,11 @@ if not (archdetect_accel == nil or archdetect_accel == '') then
         setenv("EESSI_MODULEPATH_ACCEL", eessi_module_path_accel)
         prepend_path("MODULEPATH", eessi_module_path_accel)
         eessiDebug("Using acclerator modules at: " .. eessi_module_path_accel)
+        -- do the same for site installs
+        eessi_module_path_site_accel = string.gsub(eessi_module_path_accel, "versions", "host_injections")
+        setenv("EESSI_SITE_MODULEPATH_ACCEL", eessi_module_path_site_accel)
+        prepend_path("MODULEPATH", eessi_module_path_site_accel)
+        eessiDebug("Using site acclerator modules at: " .. eessi_module_path_site_accel)
     end
 end