Lanl/mirrors

stackinator/builder.py

@@ -11,7 +11,7 @@
 import jinja2
 import yaml
 
-from . import VERSION, cache, root_logger, spack_util
+from . import VERSION, cache, root_logger, spack_util, mirror
 
 
 def install(src, dst, *, ignore=None, symlinks=False):
@@ -164,6 +164,7 @@ def environment_meta(self, recipe):
         self._environment_meta = meta
 
     def generate(self, recipe):
+        """Setup the recipe build environment."""
         # make the paths, in case bwrap is not used, directly write to recipe.mount
         store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount)
         tmp_path = self.path / "tmp"
@@ -226,12 +227,13 @@ def generate(self, recipe):
         with (self.path / "Makefile").open("w") as f:
             f.write(
                 makefile_template.render(
-                    cache=recipe.mirror,
+                    cache = recipe.cache,
                     modules=recipe.with_modules,
                     post_install_hook=recipe.post_install_hook,
                     pre_install_hook=recipe.pre_install_hook,
                     spack_version=spack_version,
                     spack_meta=spack_meta,
+                    mirrors=recipe.mirrors,
                     exclude_from_cache=["nvhpc", "cuda", "perl"],
                     verbose=False,
                 )
@@ -312,11 +314,12 @@ def generate(self, recipe):
             fid.write(global_packages_yaml)
 
         # generate a mirrors.yaml file if build caches have been configured
-        if recipe.mirror:
-            dst = config_path / "mirrors.yaml"
-            self._logger.debug(f"generate the build cache mirror: {dst}")
-            with dst.open("w") as fid:
-                fid.write(cache.generate_mirrors_yaml(recipe.mirror))
+        self._logger.debug(f"Generating the spack mirror configs in '{config_path}'")
+        try:
+            recipe.mirrors.setup_configs(config_path)
+        except mirror.MirrorError as err:
+            self._logger.error(f"Could not set up mirrors.\n{err}")
+            return 1
 
         # Add custom spack package recipes, configured via Spack repos.
         # Step 1: copy Spack repos to store_path where they will be used to

stackinator/main.py

@@ -81,13 +81,19 @@ def log_header(args):
 def make_argparser():
     parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe."))
     parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}")
-    parser.add_argument("-b", "--build", required=True, type=str)
+    parser.add_argument("-b", "--build", required=True, type=str,
+                        help="Where to set up the stackinator build directory. "
+                             "('/tmp' is not allowed, use '/var/tmp'")
     parser.add_argument("--no-bwrap", action="store_true", required=False)
-    parser.add_argument("-r", "--recipe", required=True, type=str)
-    parser.add_argument("-s", "--system", required=True, type=str)
+    parser.add_argument("-r", "--recipe", required=True, type=str,
+                        help="Name of (and/or path to) the Stackinator recipe.")
+    parser.add_argument("-s", "--system", required=True, type=str,
+                        help="Name of (and/or path to) the Stackinator system configuration.")
     parser.add_argument("-d", "--debug", action="store_true")
-    parser.add_argument("-m", "--mount", required=False, type=str)
-    parser.add_argument("-c", "--cache", required=False, type=str)
+    parser.add_argument("-m", "--mount", required=False, type=str,
+                        help="The mount point where the environment will be located.")
+    parser.add_argument("-c", "--cache", required=False, type=str, 
+                        help="Buildcache location or name (from system config's mirrors.yaml).")
     parser.add_argument("--develop", action="store_true", required=False)
 
     return parser

stackinator/mirror.py

@@ -0,0 +1,207 @@
+import base64
+import os
+import pathlib
+import urllib.request
+import urllib.error
+from typing import ByteString, Optional, List, Dict
+import magic
+
+import yaml
+
+from . import schema
+
+class MirrorError(RuntimeError):
+    """Exception class for errors thrown by mirror configuration problems."""
+
+class Mirrors:
+    """Manage the definition of mirrors in a recipe."""
+
+    KEY_STORE_DIR = 'key_store'
+    MIRRORS_YAML = 'mirrors.yaml'
+
+    def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None):
+        """Configure mirrors from both the system 'mirror.yaml' file and the command line."""
+
+        self._system_config_root = system_config_root
+
+        self.mirrors = self._load_mirrors(cmdline_cache)
+        self._check_mirrors()
+
+        self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] 
+                                   + [None]).pop(0)
+        self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)]
+        # Will hold a list of all the keys
+        self.keys = None 
+
+    def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]:
+        """Load the mirrors file, if one exists."""
+        path = self._system_config_root/"mirrors.yaml"
+        if path.exists():
+            with path.open() as fid:
+                # load the raw yaml input
+                raw = yaml.load(fid, Loader=yaml.Loader)
+
+            # validate the yaml
+            schema.CacheValidator.validate(raw)
+
+            mirrors = [mirror for mirror in raw if mirror["enabled"]]
+        else:
+            mirrors = []
+
+        buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']])
+        if buildcache_dest_count > 1:
+            raise MirrorError("Mirror config has more than one mirror specified as the build cache destination "
+                               "in the system config's 'mirrors.yaml'.")
+        elif buildcache_dest_count == 1 and cmdline_cache:
+            raise MirrorError("Build cache destination specified on the command line and in the system config's "
+                               "'mirrors.yaml'. It can be one or the other, but not both.")
+
+        # Add or set the cache given on the command line as the buildcache destination
+        if cmdline_cache is not None:
+            existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1]
+            # If the mirror name given on the command line isn't in the config, assume it 
+            # is the URL to a build cache.
+            if not existing_mirror:
+                mirrors.append(
+                    {
+                        'name': 'cmdline_cache',
+                        'url': cmdline_cache,
+                        'buildcache': True,
+                        'bootstrap': False,
+                    }
+                )
+
+        return mirrors
+
+    def _check_mirrors(self):
+        """Validate the mirror config entries."""
+
+        for mirror in self.mirrors:
+            url = mirror["url"]
+            if url.beginswith("file://"):
+                # verify that the root path exists
+                path = pathlib.Path(os.path.expandvars(url))
+                if not path.is_absolute():
+                    raise MirrorError(f"The mirror path '{path}' is not absolute")
+                if not path.is_dir():
+                    raise MirrorError(f"The mirror path '{path}' is not a directory")
+
+                mirror["url"] = path
+
+            elif url.beginswith("https://"):
+                try:
+                    request = urllib.request.Request(url, method='HEAD')
+                    urllib.request.urlopen(request)
+                except urllib.error.URLError as e:
+                    raise MirrorError(
+                        f"Could not reach the mirror url '{url}'. " 
+                        f"Check the url listed in mirrors.yaml in system config. \n{e.reason}")
+
+    def setup_configs(self, config_root: pathlib.Path):
+        """Setup all mirror configs in the given config_root."""
+
+        self._key_setup(config_root/self.KEY_STORE_DIR)
+        self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML)
+        self._create_bootstrap_configs(config_root)
+
+    def _create_spack_mirrors_yaml(self, dest: pathlib.Path):
+        """Generate the mirrors.yaml for our build directory."""
+
+        raw = {"mirrors": {}}
+
+        for m in self.mirrors:
+            name = m["name"]
+            url = m["url"]
+
+            raw["mirrors"][name] = {
+                "fetch": {"url": url},
+                "push": {"url": url},
+            }
+
+        with dest.open("w") as file:
+            yaml.dump(raw, file, default_flow_style=False)
+
+    def _create_bootstrap_configs(self, config_root: pathlib.Path):
+        """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir."""
+
+        if not self.bootstrap_mirrors:
+            return
+
+        bootstrap_yaml = {
+            'sources': [],
+            'trusted': {},
+        }
+
+        for mirror in self.bootstrap_mirrors:
+            name = mirror['name']
+            bs_mirror_path = config_root/f'bootstrap/{name}'
+            # Tell spack where to find the metadata for each bootstrap mirror.
+            bootstrap_yaml['sources'].append(
+                {
+                    'name': name,
+                    'metadata': bs_mirror_path,
+                }
+            )
+            # And trust each one
+            bootstrap_yaml['trusted'][name] = True
+
+            # Create the metadata dir and metadata.yaml
+            bs_mirror_path.mkdir(parents=True)
+            bs_mirror_yaml = {
+                'type': 'install',
+                'info': mirror['url'],
+            }
+            with (bs_mirror_path/'metadata.yaml').open('w') as file:
+                yaml.dump(bs_mirror_yaml, file, default_flow_style=False)
+
+        with (config_root/'bootstrap.yaml').open('w') as file:
+            yaml.dump(bootstrap_yaml, file, default_flow_style=False)
+
+    def _key_setup(self, key_store: pathlib.Path):
+        """Validate mirror keys, relocate to key_store, and update mirror config with new key paths."""
+
+        for mirror in self.mirrors:
+            if not mirror["public_key"]:
+                continue
+
+            key = mirror["public_key"]
+
+            # key will be saved under key_store/mirror_name.gpg
+            dest = (key_store / f"'{mirror["name"]}'.gpg").resolve()
+
+            # if path, check if abs path, if not, append sys config path in front and check again
+            path = pathlib.Path(os.path.expandvars(key))
+            if path.exists():
+                if not path.is_absolute():
+                    #try prepending system config path
+                    path = self._system_config_root/path
+                    if not path.is_file():
+                        raise MirrorError(
+                            f"The key path '{path}' is not a file. "
+                            f"Check the key listed in mirrors.yaml in system config.")
+
+                file_type = magic.from_file(path)
+
+                if not file_type.startswith("OpenPGP Public Key"):
+                    raise MirrorError(
+                        f"'{path}' is not a valid GPG key. "
+                        f"Check the key listed in mirrors.yaml in system config.")
+
+                # copy key to new destination in key store
+                with open(path, 'r') as reader, open(dest, 'w') as writer:
+                    data = reader.read()
+                    writer.write(data)
+
+            else:            
+                try:
+                    key = base64.b64decode(key)
+                except ValueError as err:
+                    pass
+                magic.from_buffer(key)
+
+                # if PGP key, convert to binary, ???, convert back
+                with open(dest, "wb") as file:
+                    file.write(key)
+
+            # update mirror with new path
+            mirror["key"] = dest

stackinator/recipe.py

@@ -4,8 +4,9 @@
 
 import jinja2
 import yaml
+from typing import Optional
 
-from . import cache, root_logger, schema, spack_util
+from . import cache, root_logger, schema, spack_util, mirror
 from .etc import envvars
 
 
@@ -169,15 +170,11 @@ def __init__(self, args):
             schema.EnvironmentsValidator.validate(raw)
             self.generate_environment_specs(raw)
 
-        # optional mirror configurtion
-        mirrors_path = self.path / "mirrors.yaml"
-        if mirrors_path.is_file():
-            self._logger.warning(
-                "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead."
-            )
-            raise RuntimeError("Unsupported mirrors.yaml file in recipe.")
-
-        self.mirror = (args.cache, self.mount)
+        # load the optional mirrors.yaml from system config, and add any additional 
+        # mirrors specified on the command line.
+        self._logger.debug("Configuring mirrors.")
+        self._mirrors = mirror.Mirrors(self.system_config_path, args.cache)
+        self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]]
 
         # optional post install hook
         if self.post_install_hook is not None:
@@ -236,32 +233,14 @@ def pre_install_hook(self):
             return hook_path
         return None
 
-    # Returns a dictionary with the following fields
-    #
-    # root: /path/to/cache
-    # path: /path/to/cache/user-environment
-    # key: /path/to/private-pgp-key
     @property
-    def mirror(self):
-        return self._mirror
-
-    # configuration is a tuple with two fields:
-    # - a Path of the yaml file containing the cache configuration
-    # - the mount point of the image
-    @mirror.setter
-    def mirror(self, configuration):
-        self._logger.debug(f"configuring build cache mirror with {configuration}")
-        self._mirror = None
-
-        file, mount = configuration
-
-        if file is not None:
-            mirror_config_path = pathlib.Path(file)
-            if not mirror_config_path.is_file():
-                raise FileNotFoundError(f"The cache configuration '{file}' is not a file")
-
-            self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount))
-
+    def mirrors(self):
+        return self._mirrors
+
+    @property
+    def cache(self):
+        return self._cache
+
     @property
     def config(self):
         return self._config
@@ -541,7 +520,7 @@ def compiler_files(self):
         )
 
         makefile_template = env.get_template("Makefile.compilers")
-        push_to_cache = self.mirror is not None
+        push_to_cache = self.cache
         files["makefile"] = makefile_template.render(
             compilers=self.compilers,
             push_to_cache=push_to_cache,
@@ -572,7 +551,7 @@ def environment_files(self):
         jenv.filters["py2yaml"] = schema.py2yaml
 
         makefile_template = jenv.get_template("Makefile.environments")
-        push_to_cache = self.mirror is not None
+        push_to_cache = self.cache is not None
         files["makefile"] = makefile_template.render(
             environments=self.environments,
             push_to_cache=push_to_cache,

stackinator/schema.py

@@ -121,3 +121,4 @@ def check_module_paths(instance):
 EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json")
 CacheValidator = SchemaValidator(prefix / "schema/cache.json")
 ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths)
+MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json")