[WIP] Compatibility layer for EESSI version 2026.x

ansible/playbooks/roles/compatibility_layer/defaults/main.yml

@@ -1,45 +1,26 @@
 # Defaults file for the compatibility layer role.
 ---
 eessi_version: "2025.06"
+eessi_host_os: "linux"
+eessi_host_arch: "x86_64"
 
 custom_overlays:
   - name: eessi
     source: git
     url: https://github.com/EESSI/gentoo-overlay.git
     eclass-overrides: true
 
 cvmfs_repository: software.eessi.io
 
 gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}
 
 # How to build the prefix.
 gentoo_git_repo: https://github.com/gentoo/gentoo.git
-# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap,
-# e.g. by checking: https://github.com/gentoo/gentoo/commits/master
-# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41
-gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7
 prefix_required_space: 15 GB
-prefix_user_defined_trusted_dirs:
-  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override"
-  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia"
-  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd"
-prefix_mask_packages: |
-  # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer,
-  # see for example https://github.com/EESSI/software-layer/issues/151
-  >=sys-devel/gcc-14
-prefix_unmask_packages: |
-  # if necessary, unmask for instance an older GCC to make it installable
-prefix_bootstrap_use_flags: |
-  # only build libnss, don't build the daemon (use the one from the host)
-  sys-auth/nss-pam-ldapd -nslcd
-  # don't build the SSSD daemon (and man pages) either
-  sys-auth/sssd -daemon -man
-  # only install Python 3.13
-  */* PYTHON_TARGETS: -* python3_13
-  */* PYTHON_SINGLE_TARGET: -* python3_13
-  # Modules: disable installation of man pages and shell setup files to be able
-  # to also install Lmod. enable *conflict unload* and *require via* features.
-  sys-cluster/modules -shell-setup -man-install +new-features
+prefix_user_defined_trusted_dirs: []
+prefix_mask_packages:
+prefix_unmask_packages:
+prefix_bootstrap_use_flags:
 prefix_use_builtin_bootstrap: false
 prefix_custom_bootstrap_script:
   local: "{{ playbook_dir }}/../../bootstrap-prefix.sh"
@@ -66,63 +47,11 @@
 package_sets:
   - "eessi-{{ eessi_version }}-{{ eessi_host_os }}-{{ eessi_host_arch }}"
 
-prefix_packages:
-
-prefix_remove_packages:
-  - app-eselect/eselect-rust
-  - dev-lang/go
-  - dev-lang/go-bootstrap
-  - dev-lang/rust
-  - dev-lang/rust-bin
-  - dev-python/setuptools-rust
-  - dev-util/cmake
-  - dev-util/hermes
-  - dev-util/ninja
-  - virtual/rust
+prefix_packages: []
+prefix_remove_packages: []
 
 reframe_venv_dir: /tmp/reframe_venv
 
 # List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION.
 # This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment.
-symlinks_to_host:
-  # required to ensure local user groups are known
-  - /etc/group
-
-  # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15)
-  - /etc/passwd
-
-  # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix
-  - /etc/hosts
-
-  # required to ensure name-service information is taken from the right source (e.g. ldap)
-  # - /etc/nsswitch.conf
-
-  # required to use the DNS resolver from the host (should be done automatically)
-  - /etc/resolv.conf
-
-  # required to pick up the right timezone from the host
-  - /etc/localtime
-
-  # required if Centrify is used in nsswitch.conf
-  - /lib64/libnss_centrifydc.so.2
-
-  # required if LDAP is used in nsswitch.conf
-  # - /lib64/libnss_ldap.so.2
-
-  # required if SSSD is used in nsswitch.conf
-  # - /lib64/libnss_sss.so.2
-
-  # required to make runtime data available for lots of tools (including who and w)
-  # - /var/run
-
-  # required by the last command
-  # - /var/log/wtmp
-
-#  - /var/lib/munge
-#  - /var/lib/unbound
-#  - /var/lib/VirtualGL
-#  - /var/log/munge
-#  - /var/spool
-#  - /var/tmp
-#  - /run/dbus
-#  - /tmp
+symlinks_to_host: []

ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml

@@ -29,9 +29,34 @@
         selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ')
       }}
 
+- name: "Clone the git repositories of git-based overlays into the prefix overlay directory"
+  ansible.builtin.git:
+    repo: "{{ item.url }}"
+    dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ item.name }}"
+    depth: 1
+    force: true
+    version: "{{ item.branch | default('HEAD') }}"
+  with_items: "{{ custom_overlays | selectattr('source', 'equalto', 'git') }}"
+
+- name: "Fetch the specific git commit to be used for the git-based overlays"
+  ansible.builtin.git:
+    repo: "{{ item.url }}"
+    dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ item.name }}"
+    clone: false
+    refspec: "{{ item.commit }}"
+    depth: 1
+    version: "{{ item.branch | default('HEAD') }}"
+
+- name: "Checkout the fetched git commit"
+  ansible.builtin.git:
+    repo: "{{ item.url }}"
+    dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ item.name }}"
+    clone: false
+    version: "{{ item.commit }}"
+
 # We do an explicit sync here (instead of using a handler),
-# so we can be sure that the overlays are up-to-date.
-- name: Sync the overlays to make sure that they are up to date
+# so we can be sure that non-git overlays are also up-to-date.
+- name: Sync overlays with auto-sync enabled to make sure that they are up to date
   community.general.portage:
     sync: 'yes'
     verbose: true

ansible/playbooks/roles/compatibility_layer/tasks/main.yml

@@ -11,6 +11,22 @@
       target CPU architecture {{ eessi_host_arch }}.
   when: not (ansible_architecture == eessi_host_arch)
 
+- name: Check for a vars file for this EESSI version
+  ansible.builtin.stat:
+    path: "{{ role_path }}/vars/{{ eessi_version }}.yml"
+  delegate_to: localhost
+  register: eessi_vars_file
+
+- name: Fail if no vars file is available for the specified/default EESSI version
+  ansible.builtin.fail:
+    msg: |
+      There is no vars file for EESSI version {{ eessi_version }} in the role's vars subdirectory.
+      Please create one and try again.
+  when: not eessi_vars_file.stat.exists
+
+- name: Include the variables file for this EESSI version
+  ansible.builtin.include_vars: "{{ role_path }}/vars/{{ eessi_version }}.yml"
+
 - name: Check if a Prefix installation is found at the specified location
   ansible.builtin.stat:
     path: "{{ gentoo_prefix_path }}/startprefix"

ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2

@@ -6,3 +6,10 @@ sync-uri = {{ item.url }}
 {% if item.branch is defined %}
 sync-git-clone-extra-opts = --branch {{ item.branch }}
 {% endif %}
+{% if item.auto-sync is defined %}
+auto-sync = {{ item.auto-sync }}
+{% elif item.source == 'git' %}
+auto-sync = no
+{% else %}
+auto-sync = yes
+{% endif %}

ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml

@@ -0,0 +1,99 @@
+# Vars file for the compatibility layer of EESSI version 2023.06
+---
+eessi_version: "2023.06"
+
+custom_overlays:
+  - name: eessi
+    source: git
+    url: https://github.com/EESSI/gentoo-overlay.git
+    eclass-overrides: true
+
+# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap,
+# e.g. by checking: https://github.com/gentoo/gentoo/commits/master
+# April 17 (29492845e41ea6a0a4a9769c7e0ce287d106079b) commit is after fix for Lmod
+# gentoo_git_commit: 29492845e41ea6a0a4a9769c7e0ce287d106079b
+# June 8 (aab8473aa90e0287553b3348a5c5b17872df4b7b) commit that was current when fetching luaposix
+gentoo_git_commit: aab8473aa90e0287553b3348a5c5b17872df4b7b
+
+prefix_user_defined_trusted_dirs:
+  - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib"
+
+prefix_mask_packages: |
+  # stick to GCC 10.x; using a too recent compiler in the compat layer complicates stuff in the software layer,
+  # see for example https://github.com/EESSI/software-layer/issues/151
+  >=sys-devel/gcc-11
+  # mask OpenSSL 3.x, stick to OpenSSL 1.1.x for now to avoid problems with:
+  # - older versions of Rust (see https://github.com/EESSI/software-layer/issues/257)
+  # - older versions of cryptograhy in Python (see https://github.com/EESSI/software-layer/issues/258)
+  >=dev-libs/openssl-3
+
+prefix_unmask_packages: |
+  # unmask older GCC to make it installable
+  =sys-devel/gcc-9*
+
+prefix_bootstrap_use_flags: |
+  # only build libnss, don't build the daemon (use the one from the host)
+  sys-auth/nss-pam-ldapd -nslcd
+  # don't build the SSSD daemon (and man pages) either
+  sys-auth/sssd -daemon -man
+  # make sure that gold linker is installed with binutils
+  sys-devel/binutils gold
+  # only install Python 3.11
+  */* PYTHON_TARGETS: -* python3_11
+  */* PYTHON_SINGLE_TARGET: -* python3_11
+
+prefix_remove_packages:
+  - app-eselect/eselect-rust
+  - dev-lang/go
+  - dev-lang/go-bootstrap
+  - dev-lang/rust
+  - dev-lang/rust-bin
+  - dev-python/setuptools-rust
+  - dev-util/cmake
+  - dev-util/ninja
+  - virtual/rust
+
+# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION.
+# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment.
+symlinks_to_host:
+  # required to ensure local user groups are known
+  - /etc/group
+
+  # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15)
+  - /etc/passwd
+
+  # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix
+  - /etc/hosts
+
+  # required to ensure name-service information is taken from the right source (e.g. ldap)
+  # - /etc/nsswitch.conf
+
+  # required to use the DNS resolver from the host (should be done automatically)
+  - /etc/resolv.conf
+
+  # required to pick up the right timezone from the host
+  - /etc/localtime
+
+  # required if Centrify is used in nsswitch.conf
+  - /lib64/libnss_centrifydc.so.2
+
+  # required if LDAP is used in nsswitch.conf
+  # - /lib64/libnss_ldap.so.2
+
+  # required if SSSD is used in nsswitch.conf
+  # - /lib64/libnss_sss.so.2
+
+  # required to make runtime data available for lots of tools (including who and w)
+  # - /var/run
+
+  # required by the last command
+  # - /var/log/wtmp
+
+#  - /var/lib/munge
+#  - /var/lib/unbound
+#  - /var/lib/VirtualGL
+#  - /var/log/munge
+#  - /var/spool
+#  - /var/tmp
+#  - /run/dbus
+#  - /tmp

ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml

@@ -0,0 +1,96 @@
+# Vars file for the compatibility layer of EESSI version 2025.06
+---
+eessi_version: "2025.06"
+
+custom_overlays:
+  - name: eessi
+    source: git
+    url: https://github.com/EESSI/gentoo-overlay.git
+    eclass-overrides: true
+
+# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap,
+# e.g. by checking: https://github.com/gentoo/gentoo/commits/master
+# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41
+gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7
+
+prefix_user_defined_trusted_dirs:
+  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override"
+  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia"
+  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd"
+
+prefix_mask_packages: |
+  # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer,
+  # see for example https://github.com/EESSI/software-layer/issues/151
+  >=sys-devel/gcc-14
+
+prefix_unmask_packages: |
+  # if necessary, unmask for instance an older GCC to make it installable
+
+prefix_bootstrap_use_flags: |
+  # only build libnss, don't build the daemon (use the one from the host)
+  sys-auth/nss-pam-ldapd -nslcd
+  # don't build the SSSD daemon (and man pages) either
+  sys-auth/sssd -daemon -man
+  # only install Python 3.13
+  */* PYTHON_TARGETS: -* python3_13
+  */* PYTHON_SINGLE_TARGET: -* python3_13
+  # Modules: disable installation of man pages and shell setup files to be able
+  # to also install Lmod. enable *conflict unload* and *require via* features.
+  sys-cluster/modules -shell-setup -man-install +new-features
+
+prefix_remove_packages:
+  - app-eselect/eselect-rust
+  - dev-lang/go
+  - dev-lang/go-bootstrap
+  - dev-lang/rust
+  - dev-lang/rust-bin
+  - dev-python/setuptools-rust
+  - dev-util/cmake
+  - dev-util/hermes
+  - dev-util/ninja
+  - virtual/rust
+
+# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION.
+# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment.
+symlinks_to_host:
+  # required to ensure local user groups are known
+  - /etc/group
+
+  # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15)
+  - /etc/passwd
+
+  # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix
+  - /etc/hosts
+
+  # required to ensure name-service information is taken from the right source (e.g. ldap)
+  # - /etc/nsswitch.conf
+
+  # required to use the DNS resolver from the host (should be done automatically)
+  - /etc/resolv.conf
+
+  # required to pick up the right timezone from the host
+  - /etc/localtime
+
+  # required if Centrify is used in nsswitch.conf
+  - /lib64/libnss_centrifydc.so.2
+
+  # required if LDAP is used in nsswitch.conf
+  # - /lib64/libnss_ldap.so.2
+
+  # required if SSSD is used in nsswitch.conf
+  # - /lib64/libnss_sss.so.2
+
+  # required to make runtime data available for lots of tools (including who and w)
+  # - /var/run
+
+  # required by the last command
+  # - /var/log/wtmp
+
+#  - /var/lib/munge
+#  - /var/lib/unbound
+#  - /var/lib/VirtualGL
+#  - /var/log/munge
+#  - /var/spool
+#  - /var/tmp
+#  - /run/dbus
+#  - /tmp