diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index 9a1afbf7..a693b822 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -1,6 +1,8 @@ # Defaults file for the compatibility layer role. --- eessi_version: "2025.06" +eessi_host_os: "linux" +eessi_host_arch: "x86_64" custom_overlays: - name: eessi @@ -14,32 +16,11 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/c # How to build the prefix. gentoo_git_repo: https://github.com/gentoo/gentoo.git -# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, -# e.g. by checking: https://github.com/gentoo/gentoo/commits/master -# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41 -gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 prefix_required_space: 15 GB -prefix_user_defined_trusted_dirs: - - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" - - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" - - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" -prefix_mask_packages: | - # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, - # see for example https://github.com/EESSI/software-layer/issues/151 - >=sys-devel/gcc-14 -prefix_unmask_packages: | - # if necessary, unmask for instance an older GCC to make it installable -prefix_bootstrap_use_flags: | - # only build libnss, don't build the daemon (use the one from the host) - sys-auth/nss-pam-ldapd -nslcd - # don't build the SSSD daemon (and man pages) either - sys-auth/sssd -daemon -man - # only install Python 3.13 - */* PYTHON_TARGETS: -* python3_13 - */* PYTHON_SINGLE_TARGET: -* python3_13 - # Modules: disable installation of man pages and shell setup files to be able - # to also install Lmod. enable *conflict unload* and *require via* features. - sys-cluster/modules -shell-setup -man-install +new-features +prefix_user_defined_trusted_dirs: [] +prefix_mask_packages: +prefix_unmask_packages: +prefix_bootstrap_use_flags: prefix_use_builtin_bootstrap: false prefix_custom_bootstrap_script: local: "{{ playbook_dir }}/../../bootstrap-prefix.sh" @@ -66,63 +47,11 @@ prefix_locales: package_sets: - "eessi-{{ eessi_version }}-{{ eessi_host_os }}-{{ eessi_host_arch }}" -prefix_packages: - -prefix_remove_packages: - - app-eselect/eselect-rust - - dev-lang/go - - dev-lang/go-bootstrap - - dev-lang/rust - - dev-lang/rust-bin - - dev-python/setuptools-rust - - dev-util/cmake - - dev-util/hermes - - dev-util/ninja - - virtual/rust +prefix_packages: [] +prefix_remove_packages: [] reframe_venv_dir: /tmp/reframe_venv # List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. # This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. -symlinks_to_host: - # required to ensure local user groups are known - - /etc/group - - # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) - - /etc/passwd - - # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix - - /etc/hosts - - # required to ensure name-service information is taken from the right source (e.g. ldap) - # - /etc/nsswitch.conf - - # required to use the DNS resolver from the host (should be done automatically) - - /etc/resolv.conf - - # required to pick up the right timezone from the host - - /etc/localtime - - # required if Centrify is used in nsswitch.conf - - /lib64/libnss_centrifydc.so.2 - - # required if LDAP is used in nsswitch.conf - # - /lib64/libnss_ldap.so.2 - - # required if SSSD is used in nsswitch.conf - # - /lib64/libnss_sss.so.2 - - # required to make runtime data available for lots of tools (including who and w) - # - /var/run - - # required by the last command - # - /var/log/wtmp - -# - /var/lib/munge -# - /var/lib/unbound -# - /var/lib/VirtualGL -# - /var/log/munge -# - /var/spool -# - /var/tmp -# - /run/dbus -# - /tmp +symlinks_to_host: [] diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index db3c6471..528143d1 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -29,9 +29,34 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} +- name: "Clone the git repositories of git-based overlays into the prefix overlay directory" + ansible.builtin.git: + repo: "{{ item.url }}" + dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ item.name }}" + depth: 1 + force: true + version: "{{ item.branch | default('HEAD') }}" + with_items: "{{ custom_overlays | selectattr('source', 'equalto', 'git') }}" + +- name: "Fetch the specific git commit to be used for the git-based overlays" + ansible.builtin.git: + repo: "{{ item.url }}" + dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ item.name }}" + clone: false + refspec: "{{ item.commit }}" + depth: 1 + version: "{{ item.branch | default('HEAD') }}" + +- name: "Checkout the fetched git commit" + ansible.builtin.git: + repo: "{{ item.url }}" + dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ item.name }}" + clone: false + version: "{{ item.commit }}" + # We do an explicit sync here (instead of using a handler), -# so we can be sure that the overlays are up-to-date. -- name: Sync the overlays to make sure that they are up to date +# so we can be sure that non-git overlays are also up-to-date. +- name: Sync overlays with auto-sync enabled to make sure that they are up to date community.general.portage: sync: 'yes' verbose: true diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml index 7a45863a..a8f935a5 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml @@ -11,6 +11,22 @@ target CPU architecture {{ eessi_host_arch }}. when: not (ansible_architecture == eessi_host_arch) +- name: Check for a vars file for this EESSI version + ansible.builtin.stat: + path: "{{ role_path }}/vars/{{ eessi_version }}.yml" + delegate_to: localhost + register: eessi_vars_file + +- name: Fail if no vars file is available for the specified/default EESSI version + ansible.builtin.fail: + msg: | + There is no vars file for EESSI version {{ eessi_version }} in the role's vars subdirectory. + Please create one and try again. + when: not eessi_vars_file.stat.exists + +- name: Include the variables file for this EESSI version + ansible.builtin.include_vars: "{{ role_path }}/vars/{{ eessi_version }}.yml" + - name: Check if a Prefix installation is found at the specified location ansible.builtin.stat: path: "{{ gentoo_prefix_path }}/startprefix" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 index 023ef327..5896e52c 100644 --- a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 +++ b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 @@ -6,3 +6,10 @@ sync-uri = {{ item.url }} {% if item.branch is defined %} sync-git-clone-extra-opts = --branch {{ item.branch }} {% endif %} +{% if item.auto-sync is defined %} +auto-sync = {{ item.auto-sync }} +{% elif item.source == 'git' %} +auto-sync = no +{% else %} +auto-sync = yes +{% endif %} diff --git a/ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml b/ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml new file mode 100644 index 00000000..ae719b6c --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml @@ -0,0 +1,99 @@ +# Vars file for the compatibility layer of EESSI version 2023.06 +--- +eessi_version: "2023.06" + +custom_overlays: + - name: eessi + source: git + url: https://github.com/EESSI/gentoo-overlay.git + eclass-overrides: true + +# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, +# e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# April 17 (29492845e41ea6a0a4a9769c7e0ce287d106079b) commit is after fix for Lmod +# gentoo_git_commit: 29492845e41ea6a0a4a9769c7e0ce287d106079b +# June 8 (aab8473aa90e0287553b3348a5c5b17872df4b7b) commit that was current when fetching luaposix +gentoo_git_commit: aab8473aa90e0287553b3348a5c5b17872df4b7b + +prefix_user_defined_trusted_dirs: + - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib" + +prefix_mask_packages: | + # stick to GCC 10.x; using a too recent compiler in the compat layer complicates stuff in the software layer, + # see for example https://github.com/EESSI/software-layer/issues/151 + >=sys-devel/gcc-11 + # mask OpenSSL 3.x, stick to OpenSSL 1.1.x for now to avoid problems with: + # - older versions of Rust (see https://github.com/EESSI/software-layer/issues/257) + # - older versions of cryptograhy in Python (see https://github.com/EESSI/software-layer/issues/258) + >=dev-libs/openssl-3 + +prefix_unmask_packages: | + # unmask older GCC to make it installable + =sys-devel/gcc-9* + +prefix_bootstrap_use_flags: | + # only build libnss, don't build the daemon (use the one from the host) + sys-auth/nss-pam-ldapd -nslcd + # don't build the SSSD daemon (and man pages) either + sys-auth/sssd -daemon -man + # make sure that gold linker is installed with binutils + sys-devel/binutils gold + # only install Python 3.11 + */* PYTHON_TARGETS: -* python3_11 + */* PYTHON_SINGLE_TARGET: -* python3_11 + +prefix_remove_packages: + - app-eselect/eselect-rust + - dev-lang/go + - dev-lang/go-bootstrap + - dev-lang/rust + - dev-lang/rust-bin + - dev-python/setuptools-rust + - dev-util/cmake + - dev-util/ninja + - virtual/rust + +# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. +# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. +symlinks_to_host: + # required to ensure local user groups are known + - /etc/group + + # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) + - /etc/passwd + + # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix + - /etc/hosts + + # required to ensure name-service information is taken from the right source (e.g. ldap) + # - /etc/nsswitch.conf + + # required to use the DNS resolver from the host (should be done automatically) + - /etc/resolv.conf + + # required to pick up the right timezone from the host + - /etc/localtime + + # required if Centrify is used in nsswitch.conf + - /lib64/libnss_centrifydc.so.2 + + # required if LDAP is used in nsswitch.conf + # - /lib64/libnss_ldap.so.2 + + # required if SSSD is used in nsswitch.conf + # - /lib64/libnss_sss.so.2 + + # required to make runtime data available for lots of tools (including who and w) + # - /var/run + + # required by the last command + # - /var/log/wtmp + +# - /var/lib/munge +# - /var/lib/unbound +# - /var/lib/VirtualGL +# - /var/log/munge +# - /var/spool +# - /var/tmp +# - /run/dbus +# - /tmp diff --git a/ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml b/ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml new file mode 100644 index 00000000..90161701 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml @@ -0,0 +1,96 @@ +# Vars file for the compatibility layer of EESSI version 2025.06 +--- +eessi_version: "2025.06" + +custom_overlays: + - name: eessi + source: git + url: https://github.com/EESSI/gentoo-overlay.git + eclass-overrides: true + +# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, +# e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41 +gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 + +prefix_user_defined_trusted_dirs: + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" + +prefix_mask_packages: | + # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, + # see for example https://github.com/EESSI/software-layer/issues/151 + >=sys-devel/gcc-14 + +prefix_unmask_packages: | + # if necessary, unmask for instance an older GCC to make it installable + +prefix_bootstrap_use_flags: | + # only build libnss, don't build the daemon (use the one from the host) + sys-auth/nss-pam-ldapd -nslcd + # don't build the SSSD daemon (and man pages) either + sys-auth/sssd -daemon -man + # only install Python 3.13 + */* PYTHON_TARGETS: -* python3_13 + */* PYTHON_SINGLE_TARGET: -* python3_13 + # Modules: disable installation of man pages and shell setup files to be able + # to also install Lmod. enable *conflict unload* and *require via* features. + sys-cluster/modules -shell-setup -man-install +new-features + +prefix_remove_packages: + - app-eselect/eselect-rust + - dev-lang/go + - dev-lang/go-bootstrap + - dev-lang/rust + - dev-lang/rust-bin + - dev-python/setuptools-rust + - dev-util/cmake + - dev-util/hermes + - dev-util/ninja + - virtual/rust + +# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. +# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. +symlinks_to_host: + # required to ensure local user groups are known + - /etc/group + + # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) + - /etc/passwd + + # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix + - /etc/hosts + + # required to ensure name-service information is taken from the right source (e.g. ldap) + # - /etc/nsswitch.conf + + # required to use the DNS resolver from the host (should be done automatically) + - /etc/resolv.conf + + # required to pick up the right timezone from the host + - /etc/localtime + + # required if Centrify is used in nsswitch.conf + - /lib64/libnss_centrifydc.so.2 + + # required if LDAP is used in nsswitch.conf + # - /lib64/libnss_ldap.so.2 + + # required if SSSD is used in nsswitch.conf + # - /lib64/libnss_sss.so.2 + + # required to make runtime data available for lots of tools (including who and w) + # - /var/run + + # required by the last command + # - /var/log/wtmp + +# - /var/lib/munge +# - /var/lib/unbound +# - /var/lib/VirtualGL +# - /var/log/munge +# - /var/spool +# - /var/tmp +# - /run/dbus +# - /tmp diff --git a/ansible/playbooks/roles/compatibility_layer/vars/2026.06.yml b/ansible/playbooks/roles/compatibility_layer/vars/2026.06.yml new file mode 100644 index 00000000..9b2a9c1f --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/vars/2026.06.yml @@ -0,0 +1,100 @@ +# Vars file for the compatibility layer of EESSI version 2026.06 +--- +eessi_version: "2026.06" +eessi_host_os: linux +eessi_host_arch: "{{ ansible_architecture }}" + +custom_overlays: + - name: eessi + source: git + url: https://github.com/EESSI/gentoo-overlay.git + eclass-overrides: true + auto-sync: false + commit: a8932a17c5d3a967a799c4ea105644ec295526e2 + +# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, +# e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41 +gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 + +prefix_user_defined_trusted_dirs: + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" + +prefix_mask_packages: | + # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, + # see for example https://github.com/EESSI/software-layer/issues/151 + >=sys-devel/gcc-14 + +prefix_unmask_packages: | + # if necessary, unmask for instance an older GCC to make it installable + +prefix_bootstrap_use_flags: | + # only build libnss, don't build the daemon (use the one from the host) + sys-auth/nss-pam-ldapd -nslcd + # don't build the SSSD daemon (and man pages) either + sys-auth/sssd -daemon -man + # only install Python 3.13 + */* PYTHON_TARGETS: -* python3_13 + */* PYTHON_SINGLE_TARGET: -* python3_13 + # Modules: disable installation of man pages and shell setup files to be able + # to also install Lmod. enable *conflict unload* and *require via* features. + sys-cluster/modules -shell-setup -man-install +new-features + +prefix_remove_packages: + - app-eselect/eselect-rust + - dev-lang/go + - dev-lang/go-bootstrap + - dev-lang/rust + - dev-lang/rust-bin + - dev-python/setuptools-rust + - dev-util/cmake + - dev-util/hermes + - dev-util/ninja + - virtual/rust + +# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. +# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. +symlinks_to_host: + # required to ensure local user groups are known + - /etc/group + + # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) + - /etc/passwd + + # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix + - /etc/hosts + + # required to ensure name-service information is taken from the right source (e.g. ldap) + # - /etc/nsswitch.conf + + # required to use the DNS resolver from the host (should be done automatically) + - /etc/resolv.conf + + # required to pick up the right timezone from the host + - /etc/localtime + + # required if Centrify is used in nsswitch.conf + - /lib64/libnss_centrifydc.so.2 + + # required if LDAP is used in nsswitch.conf + # - /lib64/libnss_ldap.so.2 + + # required if SSSD is used in nsswitch.conf + # - /lib64/libnss_sss.so.2 + + # required to make runtime data available for lots of tools (including who and w) + # - /var/run + + # required by the last command + # - /var/log/wtmp + +# - /var/lib/munge +# - /var/lib/unbound +# - /var/lib/VirtualGL +# - /var/log/munge +# - /var/spool +# - /var/tmp +# - /run/dbus +# - /tmp