Reduce the number of Spack environments (#258)

Remove a long-standing work around to an old Spack bug, that required us
to generate a separate Spack environment for every view.
This meant that if an `environments.yaml` recipe contained more than one
view in an environment (e.g. `default` and `develop`), then the
environment would be concretised twice.

Spack 1.0 fixes this issue, so we can .

This has no impact on users or recipes, it is a back end optimization.

Fixes #209

Author: bcumming

stackinator/recipe.py

@@ -269,8 +269,7 @@ def environment_view_meta(self):
         # generate the view meta data that is presented in the squashfs image meta data
         view_meta = {}
         for _, env in self.environments.items():
-            view = env["view"]
-            if view is not None:
+            for view in env["views"]:
                 view_meta[view["name"]] = {
                     "root": view["config"]["root"],
                     "activate": view["config"]["root"] + "/activate.sh",
@@ -368,58 +367,33 @@ def generate_environment_specs(self, raw):
         env_name_map = {}
         for name, config in environments.items():
             env_name_map[name] = []
-            for view, vc in config["views"].items():
-                if view in env_names:
-                    raise Exception(f"An environment view with the name '{view}' already exists.")
+            views = []
+            for view_name, vc in config["views"].items():
+                if view_name in env_names:
+                    raise Exception(f"An environment view with the name '{name}' already exists.")
+                env_names.add(view_name)
+                view_config = copy.deepcopy(vc)
                 # set some default values:
-                # vc["link"] = "roots"
-                # vc["uenv"]["add_compilers"] = True
-                # vc["uenv"]["prefix_paths"] = {}
-                if vc is None:
-                    vc = {}
-                vc.setdefault("link", "roots")
-                vc.setdefault("uenv", {})
-                vc["uenv"].setdefault("add_compilers", True)
-                vc["uenv"].setdefault("prefix_paths", {})
+                # ["link"] = "roots"
+                # ["uenv"]["add_compilers"] = True
+                # ["uenv"]["prefix_paths"] = {}
+                if view_config is None:
+                    view_config = {}
+                view_config.setdefault("link", "roots")
+                view_config.setdefault("uenv", {})
+                view_config["uenv"].setdefault("add_compilers", True)
+                view_config["uenv"].setdefault("prefix_paths", {})
                 prefix_string = ",".join(
-                    [f"{name}={':'.join(paths)}" for name, paths in vc["uenv"]["prefix_paths"].items()]
+                    [f"{pname}={':'.join(paths)}" for pname, paths in view_config["uenv"]["prefix_paths"].items()]
                 )
-                vc["uenv"]["prefix_string"] = prefix_string
-                # save a copy of the view configuration
-                env_name_map[name].append((view, vc))
-
-        # Iterate over each environment:
-        # - creating copies of the env so that there is one copy per view.
-        # - configure each view
-        for name, views in env_name_map.items():
-            numviews = len(env_name_map[name])
-
-            # The configuration of the environment without views
-            base = copy.deepcopy(environments[name])
-
-            environments[name]["view"] = None
-            for i in range(numviews):
-                # pick a name for the environment
-                cname = name if i == 0 else name + f"-{i + 1}__"
-                if i > 0:
-                    environments[cname] = copy.deepcopy(base)
-
-                view_name, view_config = views[i]
-                # note: the root path is stored as a string, not as a pathlib.PosixPath
-                # to avoid serialisation issues when generating the spack.yaml file for
-                # each environment.
-                if view_config is None:
-                    view_config = {"root": str(self.mount / "env" / view_name)}
-                else:
-                    view_config["root"] = str(self.mount / "env" / view_name)
-
-                # The "uenv" field is not spack configuration, it is additional information
-                # used by stackinator additionally set compiler paths and LD_LIBRARY_PATH
-                # Remove it from the view_config that will be passed directly to spack, and pass
-                # it separately for configuring the envvars.py helper during the uenv build.
+                view_config["uenv"]["prefix_string"] = prefix_string
+                view_config["root"] = str(self.mount / "env" / view_name)
+
                 extra = view_config.pop("uenv")
+                env_name_map[name].append((view_name, view_config))
+                views.append({"name": view_name, "config": view_config, "extra": extra})
 
-                environments[cname]["view"] = {"name": view_name, "config": view_config, "extra": extra}
+            config["views"] = views
 
         self.environments = environments
 

stackinator/templates/Makefile.environments

@@ -32,10 +32,10 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %}
 # Create environment view where requested
 {% for env, config in environments.items() %}
 {{ env }}/generated/view_config: {{ env }}/generated/env
-{% if config.view %}
-	$(SPACK) env activate --with-view default --sh ./{{ env }} > $(STORE)/env/{{ config.view.name }}/activate.sh
-	$(BUILD_ROOT)/envvars.py view {% if config.view.extra.add_compilers %}--compilers=./{{ env }}/packages.yaml {% endif %} --prefix_paths="{{ config.view.extra.prefix_string }}" $(STORE)/env/{{ config.view.name }} $(BUILD_ROOT)
-{% endif %}
+{% for view in config.views %}
+	$(SPACK) env activate --with-view {{ view.name }} --sh ./{{ env }} > $(STORE)/env/{{ view.name }}/activate.sh
+	$(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=./{{ env }}/packages.yaml {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT)
+{% endfor %}
 	touch $@
 
 {% endfor %}

stackinator/templates/environments.spack.yaml

@@ -29,10 +29,12 @@ spack:
       require: '{{ config.mpi }}'
     {% endif %}
 {% endif %}
-{% if config.view %}
+{% if config.views %}
   view:
-    default:
-      {{ config.view.config|py2yaml(6) }}
+{% for view in config.views %}
+    {{ view.name }}:
+      {{ view.config|py2yaml(6) }}
+{% endfor %}
 {% else %}
   view: false
 {% endif %}