Skip to content

Commit e2ee9ba

Browse files
Paul-FerrellPaul-Ferrell
committed
Put the mirror manipulation code in a class.
1 parent b9fe48e commit e2ee9ba

File tree

3 files changed

+134
-122
lines changed

3 files changed

+134
-122
lines changed

‎stackinator/builder.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ def environment_meta(self, recipe):
164164
self._environment_meta = meta
165165

166166
def generate(self, recipe):
167+
"""Setup the recipe build environment."""
167168
# make the paths, in case bwrap is not used, directly write to recipe.mount
168169
store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount)
169170
tmp_path = self.path / "tmp"
@@ -313,11 +314,14 @@ def generate(self, recipe):
313314

314315
# generate a mirrors.yaml file if build caches have been configured
315316
key_store = self.path / ".gnupg"
316-
if recipe.mirrors:
317-
mirror.key_setup(recipe.mirrors, config_path, key_store)
318-
dst = config_path / "mirrors.yaml"
319-
self._logger.debug(f"generate the spack mirrors.yaml: {dst}")
320-
mirror.spack_yaml_setup(recipe.mirrors, dst)
317+
mirrors = recipe.mirrors
318+
if mirrors:
319+
mirrors.key_setup(recipe.mirrors, config_path, key_store)
320+
dest = config_path / "mirrors.yaml"
321+
self._logger.debug(f"generate the spack mirrors.yaml: {dest}")
322+
mirrors.create_spack_mirrors_yaml(dest)
323+
324+
# Setup bootstrap mirror configs.
321325

322326
# Add custom spack package recipes, configured via Spack repos.
323327
# Step 1: copy Spack repos to store_path where they will be used to

‎stackinator/mirror.py‎

Lines changed: 124 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -1,145 +1,154 @@
11
import os
22
import pathlib
33
import urllib.request
4+
import urllib.error
45
from typing import Optional, List, Dict
56
import magic
67

78
import yaml
89

910
from . import schema
1011

11-
class MirrorConfigError(RuntimeError):
12+
class MirrorError(RuntimeError):
1213
"""Exception class for errors thrown by mirror configuration problems."""
1314

14-
15-
def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None):
16-
"""Configure mirrors from both the system 'mirror.yaml' file and the command line."""
17-
18-
path = system_config_root/"mirrors.yaml"
19-
if path.exists():
20-
with path.open() as fid:
21-
# load the raw yaml input
22-
raw = yaml.load(fid, Loader=yaml.Loader)
23-
24-
print(f"Configuring mirrors and buildcache from '{path}'")
25-
26-
# validate the yaml
27-
schema.CacheValidator.validate(raw)
28-
29-
mirrors = [mirror for mirror in raw if mirror["enabled"]]
30-
else:
31-
mirrors = []
32-
33-
buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']])
34-
if buildcache_dest_count > 1:
35-
raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination "
36-
"in the system config's 'mirrors.yaml'.")
37-
elif buildcache_dest_count == 1 and cmdline_cache:
38-
raise RuntimeError("Build cache destination specified on the command line and in the system config's "
39-
"'mirrors.yaml'. It can be one or the other, but not both.")
40-
41-
# Add or set the cache given on the command line as the buildcache destination
42-
if cmdline_cache is not None:
43-
existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1]
44-
# If the mirror name given on the command line isn't in the config, assume it
45-
# is the URL to a build cache.
46-
if not existing_mirror:
47-
mirrors.append(
48-
{
49-
'name': 'cmdline_cache',
50-
'url': cmdline_cache,
51-
'buildcache': True,
52-
'bootstrap': False,
53-
}
54-
)
55-
56-
for mirror in mirrors:
57-
url = mirror["url"]
58-
if url.beginswith("file://"):
59-
# verify that the root path exists
60-
path = pathlib.Path(os.path.expandvars(url))
61-
if not path.is_absolute():
62-
raise FileNotFoundError(f"The mirror path '{path}' is not absolute")
63-
if not path.is_dir():
64-
raise FileNotFoundError(f"The mirror path '{path}' is not a directory")
65-
66-
mirror["url"] = path
67-
68-
elif url.beginswith("https://"):
69-
try:
70-
request = urllib.request.Request(url, method='HEAD')
71-
response = urllib.request.urlopen(request)
72-
except urllib.error.URLError as e:
73-
raise MirrorConfigError(
74-
f"Could not reach the mirror url '{url}'. "
75-
f"Check the url listed in mirrors.yaml in system config. \n{e.reason}")
76-
77-
#if mirror["bootstrap"]:
78-
#make bootstrap dirs
79-
#bootstrap/<mirror name>/metadata.yaml
15+
class Mirrors:
16+
"""Manage the definition of mirrors in a recipe."""
17+
18+
def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None):
19+
"""Configure mirrors from both the system 'mirror.yaml' file and the command line."""
20+
21+
self._system_config_root = system_config_root
22+
23+
self.mirrors = self._load_mirrors(cmdline_cache)
24+
self._check_mirrors()
25+
26+
self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)]
27+
self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None]
28+
29+
def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]:
30+
"""Load the mirrors file, if one exists."""
31+
path = self._system_config_root/"mirrors.yaml"
32+
if path.exists():
33+
with path.open() as fid:
34+
# load the raw yaml input
35+
raw = yaml.load(fid, Loader=yaml.Loader)
36+
37+
# validate the yaml
38+
schema.CacheValidator.validate(raw)
39+
40+
mirrors = [mirror for mirror in raw if mirror["enabled"]]
41+
else:
42+
mirrors = []
43+
44+
buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']])
45+
if buildcache_dest_count > 1:
46+
raise MirrorError("Mirror config has more than one mirror specified as the build cache destination "
47+
"in the system config's 'mirrors.yaml'.")
48+
elif buildcache_dest_count == 1 and cmdline_cache:
49+
raise MirrorError("Build cache destination specified on the command line and in the system config's "
50+
"'mirrors.yaml'. It can be one or the other, but not both.")
51+
52+
# Add or set the cache given on the command line as the buildcache destination
53+
if cmdline_cache is not None:
54+
existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1]
55+
# If the mirror name given on the command line isn't in the config, assume it
56+
# is the URL to a build cache.
57+
if not existing_mirror:
58+
mirrors.append(
59+
{
60+
'name': 'cmdline_cache',
61+
'url': cmdline_cache,
62+
'buildcache': True,
63+
'bootstrap': False,
64+
}
65+
)
8066

8167
return mirrors
8268

69+
def _check_mirrors(self):
70+
"""Validate the mirror config entries."""
8371

84-
def spack_yaml_setup(mirrors, config_path):
85-
"""Generate the mirrors.yaml for spack"""
72+
for mirror in self.mirrors:
73+
url = mirror["url"]
74+
if url.beginswith("file://"):
75+
# verify that the root path exists
76+
path = pathlib.Path(os.path.expandvars(url))
77+
if not path.is_absolute():
78+
raise MirrorError(f"The mirror path '{path}' is not absolute")
79+
if not path.is_dir():
80+
raise MirrorError(f"The mirror path '{path}' is not a directory")
8681

87-
dst = config_path / "mirrors.yaml"
82+
mirror["url"] = path
8883

89-
yaml = {"mirrors": {}}
84+
elif url.beginswith("https://"):
85+
try:
86+
request = urllib.request.Request(url, method='HEAD')
87+
urllib.request.urlopen(request)
88+
except urllib.error.URLError as e:
89+
raise MirrorError(
90+
f"Could not reach the mirror url '{url}'. "
91+
f"Check the url listed in mirrors.yaml in system config. \n{e.reason}")
9092

91-
for m in mirrors:
92-
name = m["name"]
93-
url = m["url"]
93+
def create_spack_mirrors_yaml(self, dest: pathlib.Path):
94+
"""Generate the mirrors.yaml for our build directory."""
9495

95-
yaml["mirrors"][name] = {
96-
"fetch": {"url": url},
97-
"push": {"url": url},
98-
}
96+
raw = {"mirrors": {}}
9997

100-
with dst.open("w") as file:
101-
yaml.dump(yaml, default_flow_style=False)
98+
for m in self.mirrors:
99+
name = m["name"]
100+
url = m["url"]
102101

103-
# return dst
102+
raw["mirrors"][name] = {
103+
"fetch": {"url": url},
104+
"push": {"url": url},
105+
}
104106

107+
with dest.open("w") as file:
108+
yaml.dump(raw, file, default_flow_style=False)
105109

106-
def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path):
107-
"""Validate mirror keys, relocate to key_store, and update mirror config with new key paths"""
110+
def bootstrap_setup(self, config_root: pathlib.Path):
111+
"""Create the bootstrap.yaml and bootstrap metadata dirs in our build dir."""
108112

109-
for mirror in mirrors:
110-
if mirror["key"]:
111-
key = mirror["key"]
112113

113-
# key will be saved under key_store/mirror_name.gpg
114-
dst = (key_store / f"'{mirror["name"]}'.gpg").resolve()
115114

116-
# if path, check if abs path, if not, append sys config path in front and check again
117-
path = pathlib.Path(os.path.expandvars(key))
118-
if path.exists():
119-
if not path.is_absolute():
120-
#try prepending system config path
121-
path = system_config_path + path
122-
if not path.is_file():
123-
raise FileNotFoundError(
124-
f"The key path '{path}' is not a file. "
125-
f"Check the key listed in mirrors.yaml in system config.")
115+
def key_setup(self, key_store: pathlib.Path):
116+
"""Validate mirror keys, relocate to key_store, and update mirror config with new key paths."""
126117

127-
file_type = magic.from_file(path)
118+
for mirror in self.mirrors:
119+
if mirror["key"]:
120+
key = mirror["key"]
128121

129-
if not file_type.startswith("OpenPGP Public Key"):
130-
raise MirrorConfigError(
131-
f"'{path}' is not a valid GPG key. "
132-
f"Check the key listed in mirrors.yaml in system config.")
133-
134-
# copy key to new destination in key store
135-
with open(path, 'r') as reader, open(dst, 'w') as writer:
136-
data = reader.read()
137-
writer.write(data)
122+
# key will be saved under key_store/mirror_name.gpg
123+
dest = (key_store / f"'{mirror["name"]}'.gpg").resolve()
124+
125+
# if path, check if abs path, if not, append sys config path in front and check again
126+
path = pathlib.Path(os.path.expandvars(key))
127+
if path.exists():
128+
if not path.is_absolute():
129+
#try prepending system config path
130+
path = self._system_config_root/path
131+
if not path.is_file():
132+
raise MirrorError(
133+
f"The key path '{path}' is not a file. "
134+
f"Check the key listed in mirrors.yaml in system config.")
135+
136+
file_type = magic.from_file(path)
137+
138+
if not file_type.startswith("OpenPGP Public Key"):
139+
raise MirrorError(
140+
f"'{path}' is not a valid GPG key. "
141+
f"Check the key listed in mirrors.yaml in system config.")
142+
143+
# copy key to new destination in key store
144+
with open(path, 'r') as reader, open(dest, 'w') as writer:
145+
data = reader.read()
146+
writer.write(data)
147+
148+
else:
149+
# if PGP key, convert to binary, ???, convert back
150+
with open(dest, "w") as file:
151+
file.write(key)
138152

139-
else:
140-
# if PGP key, convert to binary, ???, convert back
141-
with open(dst, "w") as file:
142-
file.write(key)
143-
144-
# update mirror with new path
145-
mirror["key"] = dst
153+
# update mirror with new path
154+
mirror["key"] = dest

‎stackinator/recipe.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,8 @@ def __init__(self, args):
172172

173173
# load the optional mirrors.yaml from system config, and add any additional
174174
# mirrors specified on the command line.
175-
self._mirrors = None
176175
self._logger.debug("Configuring mirrors.")
177-
self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache)
176+
self._mirrors = mirror.Mirrors(self.system_config_path, args.cache)
178177

179178
# optional post install hook
180179
if self.post_install_hook is not None:

0 commit comments

Comments
 (0)