From f1618bfa2631cf6eda75da89e71891ba300e97ac Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 17 Feb 2026 16:46:01 -0700 Subject: [PATCH 01/24] added spack source mirrors capability, needs testing --- stackinator/builder.py | 2 ++ stackinator/schema/config.json | 7 +++++++ stackinator/templates/Makefile | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..c3de44db 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -232,6 +232,8 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + # pass source_mirrors to Makefile render + source_mirrors=recipe.config.get("source_mirrors", {}), exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d6fec3a0..4b91011d 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,6 +64,13 @@ } } }, + "source_mirrors" : { + "type" : "object", + "additionalProperties": { + "type" : "string" + }, + "default": {} + }, "modules" : { "type": "boolean" }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..3da2e00e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -39,6 +39,16 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} + {% if source_mirrors %} + echo "Replacing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Adding mirrors" + {% for name, url in source_mirrors.items() %} + $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + {% endfor %} + echo "Current mirror list:" + spack mirror list + {% endif %} touch mirror-setup compilers: mirror-setup From 954a6901a6544ef6e190bea4cdfc8ada4b6acffe Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 18 Feb 2026 14:34:05 -0700 Subject: [PATCH 02/24] removed lanl stuff --- stackinator/templates/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3da2e00e..fcf6d68a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,13 +40,13 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Replacing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Removing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' echo "Adding mirrors" {% for name, url in source_mirrors.items() %} $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} {% endfor %} - echo "Current mirror list:" + echo "Spack mirrors for this recipe:" spack mirror list {% endif %} touch mirror-setup From e2c646267a6e61a41ebe64b0b538609897ac4423 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 10:43:33 -0700 Subject: [PATCH 03/24] add source mirrors via config.yaml and retain spack default mirror --- stackinator/templates/Makefile | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68a..1d94c15a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -34,22 +34,20 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} - {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} - {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list - {% endif %} - touch mirror-setup + $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% if cache.key %} + $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + {% endif %} + {% endif %} + {% if source_mirrors %} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + touch mirror-setup compilers: mirror-setup $(SANDBOX) $(MAKE) -C $@ From 69f9bb69df6fc700cc23d1976ef729059c1f74fd Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 16:04:05 -0700 Subject: [PATCH 04/24] fixed spaces/tabs typo --- stackinator/templates/Makefile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 1d94c15a..ca8fb311 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,22 +32,22 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - + {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} - {% endfor %} - @echo "Current mirror list:" - $(SANDBOX) $(SPACK) mirror list - {% endif %} - touch mirror-setup + $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% if cache.key %} + $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + {% endif %} + {% endif %} + {% if source_mirrors %} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + touch mirror-setup compilers: mirror-setup $(SANDBOX) $(MAKE) -C $@ From 8b392c258eb75249eefbb89d5858f84a34d4416f Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:16:51 -0700 Subject: [PATCH 05/24] Added mirror configuration json schema. --- stackinator/schema/config.json | 7 ------- stackinator/schema/mirror.json | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 stackinator/schema/mirror.json diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 4b91011d..d6fec3a0 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,13 +64,6 @@ } } }, - "source_mirrors" : { - "type" : "object", - "additionalProperties": { - "type" : "string" - }, - "default": {} - }, "modules" : { "type": "boolean" }, diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..b32a3cd1 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,37 @@ +# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) +{ + # Order matters, so we need an array. + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"] + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax.", + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled.", + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + } + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages.", + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} From dba9698cd781953a4fe1d938f763ec303c588986 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:48:06 -0700 Subject: [PATCH 06/24] Incorporating Makefile changes. --- stackinator/templates/Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index ca8fb311..d1a06dd4 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,21 +32,25 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% if mirrors %} + @echo "Adding mirrors and gpg keys." + {% for mirror_info in mirrors | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} + $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} + {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} + $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} + {% endfor %} touch mirror-setup compilers: mirror-setup From b0a8071eb668af9712ad21c7b1e11b2f32306fd2 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:05:24 -0700 Subject: [PATCH 07/24] mirrors --- stackinator/mirror.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stackinator/mirror.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..e69de29b From 6a044ae4ffaee02c903675edfb4f469d16ff371a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:26:59 -0700 Subject: [PATCH 08/24] mirrors --- stackinator/mirror.py | 58 +++++++++++++++++++++++++++++++++++++++++++ stackinator/recipe.py | 32 ++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e69de29b..83989b83 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -0,0 +1,58 @@ +import os +import pathlib + +import yaml + +from . import schema + + +def configuration_from_file(file, mount): + with file.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(raw["root"])) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + raw["root"] = path + + # Put the build cache in a sub-directory named after the mount point. + # This avoids relocation issues. + raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) + + # verify that the key file exists if it was specified + key = raw["key"] + if key is not None: + key = pathlib.Path(os.path.expandvars(key)) + if not key.is_absolute(): + raise FileNotFoundError(f"The build cache key '{key}' is not absolute") + if not key.is_file(): + raise FileNotFoundError(f"The build cache key '{key}' does not exist") + raw["key"] = key + + return raw + + +def generate_mirrors_yaml(config): + path = config["path"].as_posix() + mirrors = { + "mirrors": { + "alpscache": { + "fetch": { + "url": f"file://{path}", + }, + "push": { + "url": f"file://{path}", + }, + } + } + } + + return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..d0ec018f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,14 +170,20 @@ def __init__(self, args): self.generate_environment_specs(raw) # optional mirror configurtion + if mirrors_path.is_file():zx mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): self._logger.warning( "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." ) raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - self.mirror = (args.cache, self.mount) + # self.mirror = (args.cache, self.mount) + + # load the optional mirrors.yaml from system config: + mirrors_path = self.system_config_path / "mirrors.yaml" + if mirrors_path.is_file(): + self.mirrors = (mirrors_path, self.mount) + # update mirror setter and cache.configuration_from_file() # optional post install hook if self.post_install_hook is not None: @@ -262,6 +268,28 @@ def mirror(self, configuration): self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + @property + def mirrors(self): + return self._mirrors + + # old: self.mirror = (args.cache, self.mount) + # new: self.mirror = (mirrors_yaml_path, self.mount) + + @mirrors.setter + def (self, configuration): + self._logger.debug(f"configuring mirrors with {configuration}") + self._mirrors = None + + file, mount = configuration + + if file is not None: + mirror_config_path = pathlib.Path(file) + if not mirror_config_path.is_file(): + raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") + + self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + + @property def config(self): return self._config From 0ad5022265a1b95081202aea6204d6166649c7c6 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:40:10 -0700 Subject: [PATCH 09/24] validate mirror config --- stackinator/mirror.py | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 83989b83..be784864 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,5 +1,6 @@ import os import pathlib +import urllib.request import yaml @@ -14,30 +15,28 @@ def configuration_from_file(file, mount): # validate the yaml schema.CacheValidator.validate(raw) - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw + mirrors = [mirror for mirror in raw if mirror["enabled"]] + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') + + return mirrors def generate_mirrors_yaml(config): From 3036013f07303966ac536b59b510aa7f0f7ec64c Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:01:41 -0700 Subject: [PATCH 10/24] Updating recipe to handle new mirrors format. --- stackinator/mirror.py | 4 +-- stackinator/recipe.py | 72 ++++++++++--------------------------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index be784864..08c303bb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -7,7 +7,7 @@ from . import schema -def configuration_from_file(file, mount): +def configuration_from_file(file): with file.open() as fid: # load the raw yaml input raw = yaml.load(fid, Loader=yaml.Loader) @@ -54,4 +54,4 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file + return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index d0ec018f..281ab8b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,21 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - if mirrors_path.is_file():zx - mirrors_path = self.path / "mirrors.yaml" - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - # self.mirror = (args.cache, self.mount) + mirrors_path = self.system_config_path/'mirrors.yaml' + self._logger.debug(f"opening {mirrors_path}") # load the optional mirrors.yaml from system config: - mirrors_path = self.system_config_path / "mirrors.yaml" - if mirrors_path.is_file(): - self.mirrors = (mirrors_path, self.mount) - # update mirror setter and cache.configuration_from_file() + self.mirrors = self.system_config_path / "mirrors.yaml" # optional post install hook if self.post_install_hook is not None: @@ -242,54 +233,25 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror + def mirrors(self): + return self._mirrors # configuration is a tuple with two fields: # - a Path of the yaml file containing the cache configuration # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - @property - def mirrors(self): - return self._mirrors - - # old: self.mirror = (args.cache, self.mount) - # new: self.mirror = (mirrors_yaml_path, self.mount) - @mirrors.setter - def (self, configuration): - self._logger.debug(f"configuring mirrors with {configuration}") + def mirrors(self, path: Optional[pathlib.Path]): + """Initialize the mirrors property from config.""" self._mirrors = None + if path is not None: + if not path.is_file(): + raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " + "readable file.") - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") - - self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - + self._logger.debug(f"configuring mirrors from {path}") + self._mirrors = mirror.configuration_from_file(path) + @property def config(self): return self._config @@ -569,7 +531,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.mirrors files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, From a1c486d7c9d6801533f544f0b6780e474392c3aa Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:35:42 -0700 Subject: [PATCH 11/24] Updating mirror configuration more. --- stackinator/main.py | 16 ++++--- stackinator/mirror.py | 79 +++++++++++++++++++++++----------- stackinator/recipe.py | 27 +++--------- stackinator/schema/mirror.json | 7 ++- 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..6f30ddfc 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 08c303bb..40487f48 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,42 +1,73 @@ import os import pathlib import urllib.request +from typing import Optional import yaml from . import schema -def configuration_from_file(file): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) +def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + print(f"Configuring mirrors and buildcache from '{path}'") # validate the yaml schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise RuntimeError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - mirror["url"] = path - - else: - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - return mirrors + return mirrors def generate_mirrors_yaml(config): diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 281ab8b2..4b7c038c 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,11 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - mirrors_path = self.system_config_path/'mirrors.yaml' - self._logger.debug(f"opening {mirrors_path}") - - # load the optional mirrors.yaml from system config: - self.mirrors = self.system_config_path / "mirrors.yaml" + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._mirrors = None + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) # optional post install hook if self.post_install_hook is not None: @@ -236,22 +236,7 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirrors.setter - def mirrors(self, path: Optional[pathlib.Path]): - """Initialize the mirrors property from config.""" - self._mirrors = None - if path is not None: - if not path.is_file(): - raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " - "readable file.") - - self._logger.debug(f"configuring mirrors from {path}") - self._mirrors = mirror.configuration_from_file(path) - + @property def config(self): return self._config diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index b32a3cd1..a53cc34e 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -23,7 +23,12 @@ "type": "boolean", "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", - } + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, "public_key": { "type": "string", "description": "Public PGP key for validating binary cache packages.", From 2b606830c4daf0305b19fe7587766cd4c034353b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:25:37 -0700 Subject: [PATCH 12/24] mirror yaml generator --- stackinator/mirror.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 40487f48..315d2e8e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -70,19 +70,16 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N return mirrors -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } +def generate_mirrors_yaml(mirrors): + yaml = {"mirrors": {}} + + for m in mirrors: + name = m["name"] + url = m["url"] + + yaml["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, } - } - return yaml.dump(mirrors, default_flow_style=False) + return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file From 095862c37402394424991963f83ed7dbe6bca896 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:37:34 -0700 Subject: [PATCH 13/24] update mirrors --- stackinator/builder.py | 10 ++++------ stackinator/mirror.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index c3de44db..5b223ffe 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -226,14 +226,12 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - # pass source_mirrors to Makefile render - source_mirrors=recipe.config.get("source_mirrors", {}), + mirrors=recipe.mirrors exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -314,11 +312,11 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: + if recipe.mirrors: dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 315d2e8e..e802018a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -67,7 +67,20 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N except urllib.error.URLError as e: print(f'Error: {e.reason}') - return mirrors + if mirror["key"]: + #if path, check if exists + path = pathlib.Path(os.path.expandvars(mirror["key"])) + if path.exists(): + if not path.is_file(): + raise FileNotFoundError(f"The key path '{path}' is not a file") + else + #if key, save to file, change to path + + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml + + return mirrors def generate_mirrors_yaml(mirrors): From 31a16f1d6550a545c625d779afdacc138cfa1e68 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:54:59 -0700 Subject: [PATCH 14/24] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 74 ++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e802018a..afbacafb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,15 +2,21 @@ import pathlib import urllib.request from typing import Optional +import magic import yaml from . import schema +class MirrorConfigError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" -def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + + +def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + path = system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input @@ -54,36 +60,33 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' does not exist") mirror["url"] = path - else: + elif url.beginswith("https://"): try: request = urllib.request.Request(url, method='HEAD') response = urllib.request.urlopen(request) except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - if mirror["key"]: - #if path, check if exists - path = pathlib.Path(os.path.expandvars(mirror["key"])) - if path.exists(): - if not path.is_file(): - raise FileNotFoundError(f"The key path '{path}' is not a file") - else - #if key, save to file, change to path + raise MirrorConfigError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml return mirrors -def generate_mirrors_yaml(mirrors): +def setup(mirrors, config_path): + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") + with dst.open("w") as fid: + fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -95,4 +98,37 @@ def generate_mirrors_yaml(mirrors): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file + return yaml.dump(yaml, default_flow_style=False) + +#called from builder +def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + for mirror in mirrors: + if mirror["key"]: + key = mirror["key"] + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = system_config_path + path + if not.path.is_file() + raise FileNotFoundError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorConfigError( + f"'{key}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + # copy file to key store + with file open: + data = key.read + dest = mkdir(new_key_file) + dest.write(data) + # mirror["key"] = new_path + + else: + # if PGP key, convert to binary, ???, convert back + # if key, save to file, change to path + + \ No newline at end of file From 9ef113dca30631628a59da1f3e814e230579de3e Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:57:53 -0700 Subject: [PATCH 15/24] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 44 +++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index afbacafb..e521b597 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -12,7 +12,6 @@ class MirrorConfigError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -62,7 +61,7 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt if not path.is_absolute(): raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' is not a directory") mirror["url"] = path @@ -82,11 +81,12 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt return mirrors -def setup(mirrors, config_path): +def yaml_setup(mirrors, config_path): + """Generate the mirrors.yaml for spack""" + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -98,13 +98,22 @@ def setup(mirrors, config_path): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) + with dst.open("w") as file: + yaml.dump(yaml, default_flow_style=False) + + # return dst + -#called from builder def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + for mirror in mirrors: if mirror["key"]: key = mirror["key"] + + # key will be saved under key_store/mirror_name.gpg + dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() + # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if path.exists(): @@ -115,20 +124,23 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): raise MirrorConfigError( - f"'{key}' is not a valid GPG key. " + f"'{path}' is not a valid GPG key. " f"Check the key listed in mirrors.yaml in system config.") - # copy file to key store - with file open: - data = key.read - dest = mkdir(new_key_file) - dest.write(data) - # mirror["key"] = new_path + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dst, 'w') as writer: + data = reader.read() + writer.write(data) else: # if PGP key, convert to binary, ???, convert back - # if key, save to file, change to path + with open(dst, "w") as file: + file.write(key) - \ No newline at end of file + # update mirror with new path + mirror["key"] = dst \ No newline at end of file From b9fe48e4af23f682a3360ce50300a8ac1cc8f4ea Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 9 Mar 2026 14:33:15 -0600 Subject: [PATCH 16/24] connecting mirrors to builder.py --- stackinator/builder.py | 9 +++++---- stackinator/mirror.py | 9 ++++----- stackinator/recipe.py | 2 +- stackinator/schema/mirror.json | 16 +++++++--------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5b223ffe..7918cb24 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -231,7 +231,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +312,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured + key_store = self.path / ".gnupg" if recipe.mirrors: + mirror.key_setup(recipe.mirrors, config_path, key_store) dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) + mirror.spack_yaml_setup(recipe.mirrors, dst) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e521b597..0e5ee5d1 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,7 +1,7 @@ import os import pathlib import urllib.request -from typing import Optional +from typing import Optional, List, Dict import magic import yaml @@ -74,19 +74,18 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: + #if mirror["bootstrap"]: #make bootstrap dirs #bootstrap//metadata.yaml return mirrors -def yaml_setup(mirrors, config_path): +def spack_yaml_setup(mirrors, config_path): """Generate the mirrors.yaml for spack""" dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") yaml = {"mirrors": {}} for m in mirrors: @@ -120,7 +119,7 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: if not path.is_absolute(): #try prepending system config path path = system_config_path + path - if not.path.is_file() + if not path.is_file(): raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4b7c038c..c59790b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,7 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) + self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a53cc34e..a8be6ab3 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,28 +1,26 @@ -# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) { - # Order matters, so we need an array. "type" : "array", "items": { "type": "object", - "required": ["name", "url"] + "required": ["name", "url"], "properties": { "name": { "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax.", + "description": "The name of this mirror. Should be follow standard variable naming syntax." }, "url": { "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, "enabled": { "type": "boolean", "default": true, - "description": "Whether this mirror is enabled.", + "description": "Whether this mirror is enabled." }, "bootstrap": { "type": "boolean", "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, "buildcache": { "type": "boolean", @@ -31,7 +29,7 @@ }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages.", + "description": "Public PGP key for validating binary cache packages." }, "description": { "type": "string", @@ -39,4 +37,4 @@ } } } -} +} \ No newline at end of file From e2ee9ba4a12cebeddda5193a7b1978a74b8513c5 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 13:42:30 -0600 Subject: [PATCH 17/24] Put the mirror manipulation code in a class. --- stackinator/builder.py | 14 ++- stackinator/mirror.py | 239 +++++++++++++++++++++-------------------- stackinator/recipe.py | 3 +- 3 files changed, 134 insertions(+), 122 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 7918cb24..b35545b0 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -164,6 +164,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -313,11 +314,14 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured key_store = self.path / ".gnupg" - if recipe.mirrors: - mirror.key_setup(recipe.mirrors, config_path, key_store) - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - mirror.spack_yaml_setup(recipe.mirrors, dst) + mirrors = recipe.mirrors + if mirrors: + mirrors.key_setup(recipe.mirrors, config_path, key_store) + dest = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dest}") + mirrors.create_spack_mirrors_yaml(dest) + + # Setup bootstrap mirror configs. # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 0e5ee5d1..bccd5422 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,6 +1,7 @@ import os import pathlib import urllib.request +import urllib.error from typing import Optional, List, Dict import magic @@ -8,138 +9,146 @@ from . import schema -class MirrorConfigError(RuntimeError): +class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - -def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): - """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" - - path = system_config_root/"mirrors.yaml" - if path.exists(): - with path.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - print(f"Configuring mirrors and buildcache from '{path}'") - - # validate the yaml - schema.CacheValidator.validate(raw) - - mirrors = [mirror for mirror in raw if mirror["enabled"]] - else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise RuntimeError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") - - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', - 'url': cmdline_cache, - 'buildcache': True, - 'bootstrap': False, - } - ) - - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The mirror path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' is not a directory") - - mirror["url"] = path - - elif url.beginswith("https://"): - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - raise MirrorConfigError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - - #if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml +class Mirrors: + """Manage the definition of mirrors in a recipe.""" + + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + self._system_config_root = system_config_root + + self.mirrors = self._load_mirrors(cmdline_cache) + self._check_mirrors() + + self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + + def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + """Load the mirrors file, if one exists.""" + path = self._system_config_root/"mirrors.yaml" + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise MirrorError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) return mirrors + def _check_mirrors(self): + """Validate the mirror config entries.""" -def spack_yaml_setup(mirrors, config_path): - """Generate the mirrors.yaml for spack""" + for mirror in self.mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' is not a directory") - dst = config_path / "mirrors.yaml" + mirror["url"] = path - yaml = {"mirrors": {}} + elif url.beginswith("https://"): + try: + request = urllib.request.Request(url, method='HEAD') + urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - for m in mirrors: - name = m["name"] - url = m["url"] + def create_spack_mirrors_yaml(self, dest: pathlib.Path): + """Generate the mirrors.yaml for our build directory.""" - yaml["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } + raw = {"mirrors": {}} - with dst.open("w") as file: - yaml.dump(yaml, default_flow_style=False) + for m in self.mirrors: + name = m["name"] + url = m["url"] - # return dst + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + with dest.open("w") as file: + yaml.dump(raw, file, default_flow_style=False) -def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + def bootstrap_setup(self, config_root: pathlib.Path): + """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - for mirror in mirrors: - if mirror["key"]: - key = mirror["key"] - # key will be saved under key_store/mirror_name.gpg - dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = system_config_path + path - if not path.is_file(): - raise FileNotFoundError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") + def key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - file_type = magic.from_file(path) + for mirror in self.mirrors: + if mirror["key"]: + key = mirror["key"] - if not file_type.startswith("OpenPGP Public Key"): - raise MirrorConfigError( - f"'{path}' is not a valid GPG key. " - f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dst, 'w') as writer: - data = reader.read() - writer.write(data) + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): + raise MirrorError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + # if PGP key, convert to binary, ???, convert back + with open(dest, "w") as file: + file.write(key) - else: - # if PGP key, convert to binary, ???, convert back - with open(dst, "w") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dst \ No newline at end of file + # update mirror with new path + mirror["key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index c59790b2..b03d7510 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,9 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. - self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) + self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: From 798a21651c6baa826dcc4f939aac890a20502b1a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:04:10 -0600 Subject: [PATCH 18/24] preserve cache for makefile --- stackinator/builder.py | 1 + stackinator/recipe.py | 9 +++++++-- stackinator/schema.py | 1 + stackinator/templates/Makefile | 21 ++++++++------------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b35545b0..2a2ceefb 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,6 +227,7 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( + cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, diff --git a/stackinator/recipe.py b/stackinator/recipe.py index b03d7510..9915bf91 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,6 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] # optional post install hook if self.post_install_hook is not None: @@ -235,6 +236,10 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors + + @property + def cache(self): + return self._cache @property def config(self): @@ -515,7 +520,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirrors + push_to_cache = self.cache files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -546,7 +551,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None + push_to_cache = self.cache is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d461ff0e 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d1a06dd4..9484e3c7 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,25 +32,20 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} + {% if cache %} - # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} {% endif %} {% if mirrors %} - @echo "Adding mirrors and gpg keys." - {% for mirror_info in mirrors | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} - $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} + @echo "Adding mirror gpg keys." + {% for mirror in mirrors | reverse %} + {% if mirror.public_key %} + $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} + {% endif %} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} - {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} - $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} - {% endfor %} touch mirror-setup compilers: mirror-setup @@ -89,14 +84,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if cache %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 64ff5d2a547b61561d5728ed82444617795bbb55 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:09:46 -0600 Subject: [PATCH 19/24] Adding bootstrap mirror configs. --- stackinator/builder.py | 17 ++++++++-------- stackinator/mirror.py | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2a2ceefb..acc2d91b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,15 +314,14 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - key_store = self.path / ".gnupg" - mirrors = recipe.mirrors - if mirrors: - mirrors.key_setup(recipe.mirrors, config_path, key_store) - dest = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dest}") - mirrors.create_spack_mirrors_yaml(dest) - - # Setup bootstrap mirror configs. + if recipe.mirrors: + recipe.mirrors.key_setup(config_path) + + self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") + recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') + + # Setup bootstrap mirror configs. + recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index bccd5422..9a7dc797 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,7 +2,7 @@ import pathlib import urllib.request import urllib.error -from typing import Optional, List, Dict +from typing import ByteString, Optional, List, Dict import magic import yaml @@ -22,8 +22,10 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - - self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: @@ -107,12 +109,43 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def bootstrap_setup(self, config_root: pathlib.Path): + def create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + if not self.bootstrap_mirrors: + return + + bootstrap_yaml = { + 'sources': [], + 'trusted': {}, + } + + for mirror in self.bootstrap_mirrors: + name = mirror['name'] + bs_mirror_path = config_root/f'bootstrap/{name}' + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml['sources'].append( + { + 'name': name, + 'metadata': bs_mirror_path, + } + ) + # And trust each one + bootstrap_yaml['trusted'][name] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True) + bs_mirror_yaml = { + 'type': 'install', + 'info': mirror['url'], + } + with (bs_mirror_path/'metadata.yaml').open('w') as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + + with (config_root/'bootstrap.yaml').open('w') as file: + yaml.dump(bootstrap_yaml, file, default_flow_style=False) - - def key_setup(self, key_store: pathlib.Path): + def key_setup(self, config_root: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From d55c1525c0869fb5bd9168d45d8470db64cae73e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:16:21 -0600 Subject: [PATCH 20/24] Reverted to defining the key store path in builder. --- stackinator/builder.py | 2 +- stackinator/mirror.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index acc2d91b..80a3977b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,7 +315,7 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: - recipe.mirrors.key_setup(config_path) + recipe.mirrors.key_setup(config_path/'key_store') self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9a7dc797..9f162768 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -145,7 +145,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, config_root: pathlib.Path): + def key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From 21c507ca74bf027d2f6c893e88034ca6d721e5e7 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:22:31 -0600 Subject: [PATCH 21/24] Compressed mirror config setup into a single interface. --- stackinator/builder.py | 5 +---- stackinator/mirror.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 80a3977b..f593fb05 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,12 +315,9 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") recipe.mirrors.key_setup(config_path/'key_store') - - self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - - # Setup bootstrap mirror configs. recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9f162768..6de59937 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -15,6 +15,9 @@ class MirrorError(RuntimeError): class Mirrors: """Manage the definition of mirrors in a recipe.""" + KEY_STORE_DIR = 'key_store' + MIRRORS_YAML = 'mirrors.yaml' + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -92,7 +95,14 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - def create_spack_mirrors_yaml(self, dest: pathlib.Path): + def setup_configs(self, config_root: pathlib.Path): + """Setup all mirror configs in the given config_root.""" + + self._key_setup(config_root/self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._create_bootstrap_configs(config_root) + + def _create_spack_mirrors_yaml(self, dest: pathlib.Path): """Generate the mirrors.yaml for our build directory.""" raw = {"mirrors": {}} @@ -109,7 +119,7 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def create_bootstrap_configs(self, config_root: pathlib.Path): + def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" if not self.bootstrap_mirrors: @@ -145,7 +155,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, key_store: pathlib.Path): + def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From d33de01b8d059b08aaaa196b544c07ebf4dfd7b6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:26:49 -0600 Subject: [PATCH 22/24] Catching builder exceptions. --- stackinator/builder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f593fb05..5bab0eae 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,11 +314,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirrors: - self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") - recipe.mirrors.key_setup(config_path/'key_store') - recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - recipe.mirrors.create_bootstrap_configs(config_path) + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") + try: + recipe.mirrors.setup_configs(config_path) + except mirror.MirrorError as err: + self._logger.error(f"Could not set up mirrors.\n{err}") + return 1 # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to From 3eac13d83b584c0e87625e2f1f30e20972524844 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:33:08 -0600 Subject: [PATCH 23/24] fixing key setup --- stackinator/mirror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 6de59937..c3b2c404 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -159,8 +159,8 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["key"]: - key = mirror["key"] + if mirror["public_key"]: + key = mirror["public_key"] # key will be saved under key_store/mirror_name.gpg dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() From d31422c02db8497105b14bb487d3b70eebb284ab Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:46:21 -0600 Subject: [PATCH 24/24] In progress. --- stackinator/mirror.py | 78 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index c3b2c404..8f5e4400 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,3 +1,4 @@ +import base64 import os import pathlib import urllib.request @@ -29,7 +30,8 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + [None]).pop(0) self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + # Will hold a list of all the keys + self.keys = None def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: """Load the mirrors file, if one exists.""" @@ -159,39 +161,47 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["public_key"]: - key = mirror["public_key"] - - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() - - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + if not mirror["public_key"]: + continue + + key = mirror["public_key"] + + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - # if PGP key, convert to binary, ???, convert back - with open(dest, "w") as file: - file.write(key) + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") - # update mirror with new path - mirror["key"] = dest + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + try: + key = base64.b64decode(key) + except ValueError as err: + pass + magic.from_buffer(key) + + # if PGP key, convert to binary, ???, convert back + with open(dest, "wb") as file: + file.write(key) + + # update mirror with new path + mirror["key"] = dest