connecting mirrors to builder.py

Author: grodzki-lanl

stackinator/builder.py

@@ -11,7 +11,7 @@
 import jinja2
 import yaml
 
-from . import VERSION, cache, root_logger, spack_util
+from . import VERSION, cache, root_logger, spack_util, mirror
 
 
 def install(src, dst, *, ignore=None, symlinks=False):
@@ -231,7 +231,7 @@ def generate(self, recipe):
                     pre_install_hook=recipe.pre_install_hook,
                     spack_version=spack_version,
                     spack_meta=spack_meta,
-                    mirrors=recipe.mirrors
+                    mirrors=recipe.mirrors,
                     exclude_from_cache=["nvhpc", "cuda", "perl"],
                     verbose=False,
                 )
@@ -312,11 +312,12 @@ def generate(self, recipe):
             fid.write(global_packages_yaml)
 
         # generate a mirrors.yaml file if build caches have been configured
+        key_store = self.path / ".gnupg"
         if recipe.mirrors:
+            mirror.key_setup(recipe.mirrors, config_path, key_store)
             dst = config_path / "mirrors.yaml"
             self._logger.debug(f"generate the spack mirrors.yaml: {dst}")
-            with dst.open("w") as fid:
-                fid.write(cache.generate_mirrors_yaml(recipe.mirrors))
+            mirror.spack_yaml_setup(recipe.mirrors, dst)
 
         # Add custom spack package recipes, configured via Spack repos.
         # Step 1: copy Spack repos to store_path where they will be used to

stackinator/mirror.py

@@ -1,7 +1,7 @@
 import os
 import pathlib
 import urllib.request
-from typing import Optional
+from typing import Optional, List, Dict
 import magic
 
 import yaml
@@ -74,19 +74,18 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt
                     f"Could not reach the mirror url '{url}'. " 
                     f"Check the url listed in mirrors.yaml in system config. \n{e.reason}")
 
-        if mirror["bootstrap"]:
+        #if mirror["bootstrap"]:
             #make bootstrap dirs
             #bootstrap/<mirror name>/metadata.yaml
 
         return mirrors
 
 
-def yaml_setup(mirrors, config_path):
+def spack_yaml_setup(mirrors, config_path):
     """Generate the mirrors.yaml for spack"""
 
     dst = config_path / "mirrors.yaml"
 
-    self._logger.debug(f"generate the spack mirrors.yaml: {dst}")
     yaml = {"mirrors": {}}
 
     for m in mirrors:
@@ -120,7 +119,7 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store:
                 if not path.is_absolute():
                     #try prepending system config path
                     path = system_config_path + path
-                    if not.path.is_file()
+                    if not path.is_file():
                         raise FileNotFoundError(
                             f"The key path '{path}' is not a file. "
                             f"Check the key listed in mirrors.yaml in system config.")

stackinator/recipe.py

@@ -174,7 +174,7 @@ def __init__(self, args):
         # mirrors specified on the command line.
         self._mirrors = None
         self._logger.debug("Configuring mirrors.")
-        self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache)
+        self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache)
 
         # optional post install hook
         if self.post_install_hook is not None:

stackinator/schema/mirror.json

@@ -1,28 +1,26 @@
-# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms)
 {
-    # Order matters, so we need an array.
     "type" : "array",
     "items": {
         "type": "object",
-        "required": ["name", "url"]
+        "required": ["name", "url"],
         "properties": {
             "name": {
                 "type": "string",
-                "description": "The name of this mirror. Should be follow standard variable naming syntax.",
+                "description": "The name of this mirror. Should be follow standard variable naming syntax."
             },
             "url": {
                 "type": "string",
-                "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).",
+                "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)."
             },
             "enabled": {
                 "type": "boolean",
                 "default": true,
-                "description": "Whether this mirror is enabled.",
+                "description": "Whether this mirror is enabled."
             },
             "bootstrap": {
                 "type": "boolean",
                 "default": false,
-                "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.",
+                "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror."
             },
             "buildcache": {
                 "type": "boolean",
@@ -31,12 +29,12 @@
             },
             "public_key": {
                 "type": "string",
-                "description": "Public PGP key for validating binary cache packages.",
+                "description": "Public PGP key for validating binary cache packages."
             },
             "description": {
                 "type": "string",
                 "description": "What this mirror is for."
             }
         }
     }
-}
+}