diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..5bab0eae 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -164,6 +164,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -226,12 +227,13 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, + cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +314,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") + try: + recipe.mirrors.setup_configs(config_path) + except mirror.MirrorError as err: + self._logger.error(f"Could not set up mirrors.\n{err}") + return 1 # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..6f30ddfc 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..8f5e4400 --- /dev/null +++ b/stackinator/mirror.py @@ -0,0 +1,207 @@ +import base64 +import os +import pathlib +import urllib.request +import urllib.error +from typing import ByteString, Optional, List, Dict +import magic + +import yaml + +from . import schema + +class MirrorError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" + +class Mirrors: + """Manage the definition of mirrors in a recipe.""" + + KEY_STORE_DIR = 'key_store' + MIRRORS_YAML = 'mirrors.yaml' + + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + self._system_config_root = system_config_root + + self.mirrors = self._load_mirrors(cmdline_cache) + self._check_mirrors() + + self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] + # Will hold a list of all the keys + self.keys = None + + def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + """Load the mirrors file, if one exists.""" + path = self._system_config_root/"mirrors.yaml" + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise MirrorError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + return mirrors + + def _check_mirrors(self): + """Validate the mirror config entries.""" + + for mirror in self.mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' is not a directory") + + mirror["url"] = path + + elif url.beginswith("https://"): + try: + request = urllib.request.Request(url, method='HEAD') + urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + + def setup_configs(self, config_root: pathlib.Path): + """Setup all mirror configs in the given config_root.""" + + self._key_setup(config_root/self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._create_bootstrap_configs(config_root) + + def _create_spack_mirrors_yaml(self, dest: pathlib.Path): + """Generate the mirrors.yaml for our build directory.""" + + raw = {"mirrors": {}} + + for m in self.mirrors: + name = m["name"] + url = m["url"] + + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + + with dest.open("w") as file: + yaml.dump(raw, file, default_flow_style=False) + + def _create_bootstrap_configs(self, config_root: pathlib.Path): + """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + + if not self.bootstrap_mirrors: + return + + bootstrap_yaml = { + 'sources': [], + 'trusted': {}, + } + + for mirror in self.bootstrap_mirrors: + name = mirror['name'] + bs_mirror_path = config_root/f'bootstrap/{name}' + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml['sources'].append( + { + 'name': name, + 'metadata': bs_mirror_path, + } + ) + # And trust each one + bootstrap_yaml['trusted'][name] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True) + bs_mirror_yaml = { + 'type': 'install', + 'info': mirror['url'], + } + with (bs_mirror_path/'metadata.yaml').open('w') as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + + with (config_root/'bootstrap.yaml').open('w') as file: + yaml.dump(bootstrap_yaml, file, default_flow_style=False) + + def _key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + for mirror in self.mirrors: + if not mirror["public_key"]: + continue + + key = mirror["public_key"] + + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): + raise MirrorError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + try: + key = base64.b64decode(key) + except ValueError as err: + pass + magic.from_buffer(key) + + # if PGP key, convert to binary, ???, convert back + with open(dest, "wb") as file: + file.write(key) + + # update mirror with new path + mirror["key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..9915bf91 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,15 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - self.mirror = (args.cache, self.mount) + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] # optional post install hook if self.post_install_hook is not None: @@ -236,32 +233,14 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - + def mirrors(self): + return self._mirrors + + @property + def cache(self): + return self._cache + @property def config(self): return self._config @@ -541,7 +520,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.cache files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -572,7 +551,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None + push_to_cache = self.cache is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d461ff0e 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..a8be6ab3 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,40 @@ +{ + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"], + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax." + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled." + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages." + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} \ No newline at end of file diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..9484e3c7 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -35,9 +35,16 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} + {% if mirrors %} + @echo "Adding mirror gpg keys." + {% for mirror in mirrors | reverse %} + {% if mirror.public_key %} + $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} + {% endif %} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list {% endif %} touch mirror-setup @@ -77,14 +84,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if cache %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\