From 480105c5f3d830ef59f6e7f10e8ea6d1f87ceab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 10 Feb 2026 17:09:35 +0100 Subject: [PATCH] New PEP: Wheel Variants: Package Format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following the suggestions given in the previous PEP 817 thread [1], we have decided to split PEP 817 into a series of smaller PEPs, with the hope that this will make it easier to comprehend the concept and discuss it. This is the first split PEP, that specifically focuses on the low-level details necessary for variants wheels to work, that is: - adding variant label to the filename - storing variant properties in the file - exposing variants on the index - ordering/selecting variants - introducing variant-conditional dependencies via environment markers - exposing variant wheels in `pylock.toml` The PEP keeps variant properties abstract, deferring their governance and determining their compatibility to a subsequent PEP, along with building wheels. We've also significantly cut motivation down (the original is kept in PEP 817 for reference). We've tried to make the "specification" part easier to comprehend, and removed the duplicate "rationale-overview", in favor of a more focused "rationale" section. Compared to the previous iteration of PEP 817, we've also corrected the variant ordering algorithm to handle corner cases better. [1] https://discuss.python.org/t/pep-817-wheel-variants-beyond-platform-tags/105860 Signed-off-by: Michał Górny Co-authored-by: Jonathan Dekhtiar Co-authored-by: Konstantin Schütze Co-authored-by: Ralf Gommers --- peps/pep-9999.rst | 962 ++++++++++++++++++ .../appendix-core-metadata-json-schema.rst | 11 + peps/pep-9999/variant_schema.json | 113 ++ 3 files changed, 1086 insertions(+) create mode 100644 peps/pep-9999.rst create mode 100644 peps/pep-9999/appendix-core-metadata-json-schema.rst create mode 100644 peps/pep-9999/variant_schema.json diff --git a/peps/pep-9999.rst b/peps/pep-9999.rst new file mode 100644 index 00000000000..9557be2c362 --- /dev/null +++ b/peps/pep-9999.rst @@ -0,0 +1,962 @@ +PEP: 9999 +Title: Wheel Variants: Package Format +Author: Jonathan Dekhtiar , + Michał Górny , + Konstantin Schütze , + Ralf Gommers , + Andrey Talman , + Charlie Marsh , + Michael Sarahan , + Eli Uriegas , + Barry Warsaw , + Donald Stufft , + Andy R. Terrel +Discussions-To: Pending +Status: Draft +Type: Standards Track +Topic: Packaging +Created: 10-Feb-2026 +Post-History: + +Abstract +======== + +This PEP proposes variant wheels, an extension to +:doc:`packaging:specifications/binary-distribution-format` that permits +building multiple variants of the same package while embedding +additional compatibility data. The specific properties are stored inside +the wheel, and expressed via a human-readable variant label in the +filename, which is then mapped to the actual properties via a separately +hosted JSON mapping. This aims to make ``{tool} install {package}`` +capable of selecting the most appropriate variant of packages where +additional compatibility dimensions such as GPU support need to be +accounted for. + + +Motivation +========== + +This PEP proposes a protocol to record additional compatibility data in +binary packages, to allow tools to pick the correct package to use in +situations where +:doc:`packaging:specifications/platform-compatibility-tags` are +insufficient. There are many cases where this is necessary, most notably +in the case of scientific and machine learning (ML) libraries, where +high performance requires extension code that is carefully tailored to +the precise hardware available in the user's environment. Well known +examples of this include: + +- PyTorch and other ML tools which depend on the user's GPU hardware and + driver. +- Scientific libraries like SciPy, which can be linked to different + linear algebra libraries. +- Libraries such as XGBoost that can be linked to different OpenMP + runtimes. +- Libraries that ship performance enhanced builds which can be used when + certain CPU instruction sets are available, such as AVX2 or AVX-512. + +The problem space has been explored in greater detail in :pep:`817`. + + +Specification +============= + +Definitions +----------- + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this +document are to be interpreted as described in :rfc:`2119`. + + +Variant wheel +------------- + +A variant wheel is an extension of the wheel format, defined in +:doc:`packaging:specifications/binary-distribution-format`. It +MUST specify a `variant label`_ in the filename, which makes it +distinct from non-variant wheels. It MUST include a +`variant metadata`_ file, which maps the variant label to zero or +more `variant properties`_. + +Variant properties +------------------ + +Variant properties express the compatibility of binary packages with +specific platforms, in addition to +:doc:`packaging:specifications/platform-compatibility-tags`. They follow +a key-value format, where a key is called a *variant feature*. The +keys are further grouped into independently governed *variant +namespaces*. Hence, a variant feature consists of a namespace and a +feature name, whereas a variant property consists of a namespace, a +feature name and a feature value. + +Variant properties are serialized into a structured 3-tuple of the +following format:: + + {namespace} :: {feature_name} :: {feature_value} + +The properties with which the wheel was built are stored within the +wheel, in the `variant metadata`_ file. A variant wheel can specify +multiple values corresponding to a variant feature. For the wheel to be +considered compatible with a system, at least one value for every +feature listed in its properties MUST be compatible with the system. +A variant wheel with zero properties is always deemed compatible. + +The namespace and feature name components MUST be non-empty and consist +only of ``0-9``, ``a-z`` and ``_`` ASCII characters (``^[a-z0-9_]+$``). +The feature value MUST be non-empty and consist only of ``0-9``, +``a-z``, ``_`` and ``.`` ASCII Characters (``^[a-z0-9_.]+$``). + +The available properties and the rules governing their compatibility +will be defined in a subsequent PEP. + +Examples: + +.. code:: text + + # the system must be compatible with all of the following + x86_64 :: level :: v3 + x86_64 :: avx512_bf16 :: on + nvidia :: cuda_version_lower_bound :: 12.8 + # it must also be compatible with at least one of the following + nvidia :: sm_arch :: 120_real + nvidia :: sm_arch :: 110_real + + +Variant label +------------- + +The wheel filename template originally defined by :pep:`427` is changed +to: + +.. code:: text + + {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}(-{variant label})?.whl + +++++++++++++++++++ + +Variant wheels MUST include the variant label component. Conversely, +wheels without variant label are non-variant wheels. The variant label +MUST consist only of ``0-9``, ``a-z``, ``_`` and ``.`` ASCII characters, +and be 1-16 characters long (``^[0-9a-z_.]{1,16}$``). + +Every variant label MUST uniquely correspond to a specific set of +variant properties, which MUST be the same for all wheels using the same +label within a single package version. + +The label ``null`` is reserved and always corresponds to the variant +with zero properties, called a null variant. This variant acts as a +fallback variant that is always compatible. + +Examples: + +- Non-variant wheel: + ``numpy-2.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl`` +- Wheel with variant label ``x86_64_v3``: + ``numpy-2.3.2-cp313-cp313t-musllinux_1_2_x86_64-x86_64_v3.whl`` +- Null variant: + ``numpy-2.3.2-cp313-cp313t-musllinux_1_2_x86_64-null.whl`` + + +Variant metadata +---------------- + +The additional metadata specific to variant wheels is stored inside the +wheel, in ``*.dist-info/variant.json`` file, using the JSON format. This +PEP defines the following structure: + +.. code:: text + + +- $schema + +- default-priorities + | +- namespace : list[str] + | +- feature + | +- {namespace} : list[str] = [] + | +- property + | +- {namespace} + | +- {feature} : list[str] = [] + +- variants + +- {variant_label} + +- {namespace} + +- {feature} : list[str] = [] + +This structure corresponds to the version ``0.0.1`` of the format. The +version number is stored as part of the schema_ URL. Whenever the format +changes, the version number must be incremented. Tools MUST assume that +all variant wheels using an unknown format version are unsupported. + +The top-level keys are described in the subsequent sections. + + +Schema +'''''' + +The ``$schema`` key is the standard way of specifying the `JSON schema +`__ used. Its value MUST be the URL of a JSON +schema corresponding to this specification, hosted on +``packaging.python.org``. The schema URL MUST include a version number, +and consequently every schema MUST describe the matching format version. +The schema can be used to verify the validity of the JSON file prior to +processing it, or after outputting it. + +A proposed JSON schema for the current format version is included in the +Appendix of this PEP. Subsequent PEPs changing the metadata format will +include updated versions of the schema. The schema is available in +:ref:`9999-variant-json-schema`. + + +Default priorities +'''''''''''''''''' + +The ``default-priorities`` dictionary defines the ordering of variants. +The exact algorithm is described in the `Variant ordering`_ section. + +The following key is REQUIRED: + +- ``namespace: list[str]``: All variant namespaces used in variant + wheels for a given package version, ordered in decreasing priority. + This list MUST contain all namespaces used in variant properties. + +It MAY have the following OPTIONAL keys: + +- ``feature: dict[str, list[str]]``: A dictionary with namespaces as + keys, and ordered list of corresponding feature names as values. + The feature names are ordered in decreasing priority. It is used to + override the default feature ordering. + +- ``property: dict[str, dict[str, list[str]]]``: A nested dictionary + with namespaces as first-level keys, feature names as second-level + keys and ordered lists of corresponding property values as + second-level values. The feature values are ordered in decreasing + priority. It is used to override the default value ordering. + + +Variants +'''''''' + +The ``variants`` dictionary provides a mapping from variant labels +to variant properties. In the variant wheel, it MUST contain the label +present in that wheel's filename. + +It has 3 levels. The first level keys are variant labels, the second +level keys are namespaces, the third level are feature names, and the +third level values are lists of feature values. + + +Example +''''''' + +.. code:: json5 + + { + // The schema URL will be replaced with the final URL on packaging.python.org + "$schema": "https://variants-schema.wheelnext.dev/v0.0.3.json", + + "default-priorities": { + // REQUIRED: specifies that x86_64 CPU properties are more important than + // aarch64 CPU properties (both are mutually exclusive, so the exact order + // does not matter), and both are more important than specific BLAS/LAPACK + // library: + "namespace": ["x86_64", "aarch64", "blas_lapack"], + + // OPTIONAL: makes "library" the most important feature in "blas_lapack" + // namespace + "feature": { + "blas_lapack": ["library"] + }, + + // OPTIONAL: makes ["mkl", "openblas"] the most important values of + // "blas_lapack :: library" feature + "property": { + "blas_lapack": { + "library": ["mkl", "openblas"] + } + } + }, + + "variants": { + // REQUIRED: in variant.json, always a single entry, with the key + // matching the variant label ("x8664v3_openblas") and the value + // specifying its properties (the system must be compatible with both): + // - blas_lapack :: library :: openblas + // - x86_64 :: level :: v3 + "x8664v3_openblas": { + "blas_lapack": { + "library": ["openblas"] + }, + "x86_64": { + "level": ["v3"] + } + } + } + } + + +Index-level metadata +-------------------- + +For every package version that includes at least one variant wheel, +there MUST exist a corresponding ``{name}-{version}-variants.json`` +file, hosted and served by the package index. The ``{name}`` and +``{version}`` placeholders correspond to the package name and version, +normalized according to the same rules as wheel files, as found in the +:ref:`packaging:wheel-file-name-spec` of the Binary Distribution Format +specification. The link to this file MUST be present on all index pages +where the variant wheels are linked. It is presented in the same simple +repository format as source distribution and wheel links in the index, +including an (OPTIONAL) hash. + +This file uses the same structure as `variant metadata`_, except that +the ``variants`` object MUST list all variants available on the package +index for the package version in question. The tools MUST ensure that +the variant metadata across multiple variant wheels of the same package +version and the index-level metadata file is consistent. They MAY +require that keys other than ``variants`` have exactly the same values, +or they may carefully merge their values, provided that no conflicting +information is introduced, and the resolution results within a subset of +variants do not change. + +Variant indexes MAY elect to either auto-generate the file from the +uploaded variant wheels or allow the user to manually generate it +themselves and upload it to the index. + +The ``foo-1.2.3-variants.json`` corresponding to the package with two +wheel variants, one of them listed in the previous example, would look +like: + +.. code:: json5 + + { + // The schema URL will be replaced with the final URL on packaging.python.org + "$schema": "https://variants-schema.wheelnext.dev/v0.0.3.json", + "default-priorities": { + // identical to above + }, + "variants": { + // REQUIRED: entries for all wheel variants for the package version + + // "x8664v3_openblas" label corresponds to: + // - blas_lapack :: library :: openblas + // - x86_64 :: level :: v3 + "x8664v3_openblas": { + "blas_lapack": { + "library": ["openblas"] + }, + "x86_64": { + "level": ["v3"] + } + }, + + // "x8664v4_mkl" label corresponds to: + // - blas_lapack :: library :: mkl + // - x86_64 :: level :: v4 + "x8664v4_mkl": { + "blas_lapack": { + "library": ["mkl"] + }, + "x86_64": { + "level": ["v4"] + } + } + } + } + + +Variant ordering +---------------- + +To determine which variant wheel to install when multiple wheels are +compatible, variants MUST be totally ordered by their variant +properties. + +For the purpose of ordering, variant properties are grouped into +features, and features into namespaces. For every namespace, the tool +MUST obtain an ordered list of compatible features, and for every +feature, a list of compatible values. The method of obtaining these +lists will be defined in a subsequent PEP. + +The ordering MUST be performed equivalent to the following algorithm: + +1. Construct the ordered list of namespaces by copying the value of the + ``default-priorities.namespace`` key from `index-level metadata`_. + This is ``namespace_order`` in the example. + +2. For every namespace: + + i. Construct the initial ordered list of feature names by copying the + value of the respective ``default-priorities.feature.{namespace}`` + key. + + ii. Obtain the compatible feature names, in order. For every feature + name that is not present in the constructed list, append it to + the end. + + After this step, a list of ordered feature names is available for + every namespace. This is ``feature_order`` in the example. + +3. For every feature: + + i. Construct the initial ordered list of values by copying the value + of the respective + ``default-priorities.property.{namespace}.{feature_name}`` key. + + ii. Obtain the compatible feature values, in order. For every value + that is not present in the constructed list, append it to the + end. + + After this step, a list of ordered property values is available for + every feature. This is ``value_order`` in the example. + +4. For every compatible variant, determine the most preferred value + corresponding to every feature in that variant. This is done by + finding among the values present in the variant properties the one + that has the lowest position in the ordered property value list. + After this step, a list of features along with their best values + is available for every variant. This is done in the + ``Variant.best_value_properties()`` method in the example. + +5. For every item in the list constructed in the previous step, + construct a sort key that is a 3-tuple consisting of + its namespace, feature name and best feature value indices in the + respective ordered lists. This is done by the ``property_key()`` + function in the example. + +6. For every compatible variant, sort the list constructed in step 4 + using the sort keys constructed in step 5, in ascending order. This + is done by the ``Variant.sorted_properties()`` method in the example. + +7. To order variants, compare their sorted lists from step 6. If the + sort keys at the first position are different, the variant with the + lower key is sorted earlier. If they are the same, compare the keys + at the second position, and so on, until either a tie-breaker is + found or the list in one of the variants is exhausted. In the latter + case, the variant with more keys is sorted earlier. As a fallback, + if both variants have the same number of keys, they are ordered + lexically by their variant label, ascending. This is done by the + ultimate step of the example algorithm, with the comparison function + being implemented as ``VariantWheel.__lt__()``. + +After this process, the variant wheels are sorted from the most +preferred to the least preferred. The algorithm sorts the null variant +after all the other variants. The non-variant wheel MUST be ordered +after the null variant. Multiple wheels with the same variant property +set (and multiple non-variant wheels) MUST then be ordered according to +their platform compatibility tags. + +Alternatively, the sort algorithm for variant wheels could be described +using the following pseudocode. For simplicity, this code does not +account for non-variant wheels or tags. + +.. code:: python + + from typing import Self + + + def get_compatible_feature_names(namespace: str) -> list[str]: + """Get an ordered list of compatible features""" + ... + + + def get_compatible_feature_values(namespace: str, feature_name: str) -> list[str]: + """Get an ordered list of compatible values""" + ... + + + # default-priorities dict from index-level metadata + default_priorities = { + "namespace": [...], # : list[str] + "feature": {...}, # : dict[str, list[str]] + "property": {...}, # : dict[str, dict[str, list[str]]] + } + + # 1. Construct the ordered list of namespaces. + namespace_order = default_priorities["namespace"] + feature_order = {} + value_order = {} + + for namespace in namespace_order: + # 2. Construct the ordered lists of feature names. + feature_order[namespace] = default_priorities["feature"].get(namespace, []) + for feature_name in get_compatible_feature_names(namespace): + if feature_name not in feature_order[namespace]: + feature_order[namespace].append(feature_name) + + value_order[namespace] = {} + for feature_name in feature_order[namespace]: + # 3. Construct the ordered lists of feature values. + value_order[namespace][feature_name] = ( + default_priorities["property"].get(namespace, {}).get(feature_name, []) + ) + for feature_value in get_compatible_feature_values(namespace, feature_name): + if feature_value not in value_order[namespace][feature_name]: + value_order[namespace][feature_name].append(feature_value) + + + def best_value_property(namespace, feature_name, feature_values) -> str: + """Helper function to determine the best value for given feature""" + for best_value in value_order[namespace][feature_name]: + if best_value in feature_values: + return best_value + assert False, "No feature value supported, wheel should have been filtered out" + + + def property_key(prop: tuple[str, str, str]) -> tuple[int, int, int]: + """Construct a sort key for variant property (akin to step 5.)""" + namespace, feature_name, feature_value = prop + return ( + namespace_order.index(namespace), + feature_order[namespace].index(feature_name), + value_order[namespace][feature_name].index(feature_value), + ) + + + class Variant: + """Example class exposing properties of a variant wheel""" + + label: str + # {namespace: {feature_name: [feature_values]}}, as in variant.json + properties: dict[str, dict[str, list[str]]] + + def best_value_properties(self) -> tuple[str, str, str]: + """Determine the most preferred values for every feature, step 4.""" + return [ + ( + namespace, + feature_name, + best_value_property(namespace, feature_name, feature_values), + ) + for namespace, features in self.properties.items() + for feature_name, feature_values in features.items() + ] + + def sorted_properties(self) -> list[tuple[str, str, str]]: + """Sort the list of features with their best values (step 6.)""" + return sorted(self.best_value_properties(), key=property_key) + + def __lt__(self: Self, other: Self) -> bool: + """Variant comparison function for sorting (part of step 7.)""" + self_properties = self.sorted_properties() + other_properties = other.sorted_properties() + # Proceed from the first to the last common sort best-value property. + # If any of them are different, the variant with better property wins. + for self_prop, other_prop in zip(self_properties, other_properties): + if self_prop != other_prop: + return property_key(self_prop) < property_key(other_prop) + # If the best-value properties of one variant are a subset of another, + # the one with more properties wins. + if len(self_properties) != len(other_properties): + return len(self_properties) > len(other_properties) + # If two variants have exactly the same properties, fall back to + # sorting on variant label (they must be unique). + return self.label < other.label + + + # A list of variants to sort. + variants: list[Variant] = [...] + + + # 7. Order variants by comparing their sorted properties + # (see VariantWheel.__lt__()) + variants.sort() + + +Environment markers +------------------- + +Four new :ref:`environment markers +` are introduced in +dependency specifications: + +1. ``variant_namespaces`` corresponding to the set of namespaces of all + the variant properties that the wheel variant was built for. +2. ``variant_features`` corresponding to the set of + ``namespace :: feature`` pairs of all the variant properties that the + wheel variant was built for. +3. ``variant_properties`` corresponding to the set of + ``namespace :: feature :: value`` tuples of all the variant + properties that the wheel variant was built for. +4. ``variant_label`` corresponding to the exact variant label that the + wheel was built with. For the non-variant wheel, it is an empty + string. + +The markers evaluating to sets of strings MUST be matched via the ``in`` +or ``not in`` operator, e.g.: + +.. code:: + + # satisfied by any "foo :: * :: *" property + dep1; "foo" in variant_namespaces + # satisfied by any "foo :: bar :: *" property + dep2; "foo :: bar" in variant_features + # satisfied only by "foo :: bar :: baz" property + dep3; "foo :: bar :: baz" in variant_properties + +The ``variant_label`` marker is a plain string: + +.. code:: + + # satisfied by the variant "foobar" + dep4; variant_label == "foobar" + # satisfied by any wheel other other than the null variant + # (including the non-variant wheel) + dep5; variant_label != "null" + # satisfied by the non-variant wheel + dep6; variant_label == "" + +Implementations MUST ignore differences in whitespace while matching the +features and properties. + +Variant marker expressions MUST be evaluated against the variant +properties stored in the wheel being installed. + + +Integration with pylock.toml +---------------------------- + +Variant wheels can be listed in ``pylock.toml`` file in the same manner +as wheels with different Platform compatibility tags: either all variant +(and non-variant) wheels can be listed, or a subset of them. + +A new ``[packages.variants-json]`` subtable is added to the file, to +permit locking the ``{name}-{version}-variants.json`` file to a specific +hash. If variant wheels are listed for a given package, the tool SHOULD +lock this file as well. + +If variant wheels are listed, the tool SHOULD resolve variants to select +the best wheel file. + +The proposed text for :doc:`packaging:specifications/pylock-toml` +follows: + +.. code:: rst + + .. _pylock-packages-variants-json: + + ``[packages.variants-json]`` + ---------------------------- + + - **Type**: table + - **Required?**: no; requires that :ref:`pylock-packages-wheels` is used, + mutually-exclusive with :ref:`pylock-packages-vcs`, + :ref:`pylock-packages-directory`, and :ref:`pylock-packages-archive`. + - **Inspiration**: uv_ + - The URL or path to the ``variants.json`` file. + - Only used if the project uses :ref:`wheel variants `. + + .. _pylock-packages-variants-json-url: + + ``packages.variants-json.url`` + '''''''''''''''''''''''''''''' + + See :ref:`pylock-packages-archive-url`. + + .. _pylock-packages-variants-json-path: + + ``packages.variants-json.path`` + ''''''''''''''''''''''''''''''' + + See :ref:`pylock-packages-archive-path`. + + .. _pylock-packages-variants-json-hashes: + + ``packages.variants-json.hashes`` + ''''''''''''''''''''''''''''''''' + + See :ref:`pylock-packages-archive-hashes`. + + +Suggested implementation logic for tools (non-normative) +-------------------------------------------------------- + +Installing a package from an index +'''''''''''''''''''''''''''''''''' + +When asked to install a version of a package from an index, the proposed +behavior would be to: + +1. Query the remote index for the package in question. +2. Initially select a package version meeting the version constraints + (this does not need to take variant metadata into account). +3. Filter available wheels based on Platform Compatibility Tags. +4. Determine if any of the remaining wheels are variant wheels. + If not, proceed as with non-variant wheels. +5. If any wheels feature a `variant label`_, download the `index-level + metadata`_ file, ``{name}-{version}-variants.json``. If this + file is missing, assume all variant wheels are incompatible and + proceed as with non-variant wheels. +6. Map the variant labels into sets of variant properties using the + index-level variant metadata file. If any of the labels present in + wheel filenames are missing in the file, assume that the respective + wheels are incompatible. +7. Obtain the ordered lists of compatible variant properties. The + mechanism for this will be specified in a subsequent PEP. +8. Filter and order variants based on the lists of compatible + properties, per `variant ordering`_, and select the most preferred + variant. If no variant wheel matched, use the non-variant wheels by + their rules. +9. If multiple wheels for a given version share the same variant label, + order them by Platform compatibility tags and build number, and + select the best wheel. + +Note that steps 4. through 8. are introduced specifically for variant +wheels. The remaining steps correspond to the current installer +behavior. + + +Installing a local wheel +'''''''''''''''''''''''' + +When asked to install a local wheel file, the proposed behavior would be +to: + +1. If no variant label is present in the filename, proceed as with + non-variant wheels. +2. Verify the wheel compatibility via Platform compatibility tags. +3. Read the `variant metadata`_ from ``*.dist-info/variant.json`` inside + the wheel file. +4. Obtain the ordered lists of compatible variant properties. The + mechanism for this will be specified in a subsequent PEP. +5. Verify the wheel compatibility via compatible properties. + + +Publishing variant wheels on an index +''''''''''''''''''''''''''''''''''''' + +Variant wheels are uploaded to an index just like regular wheels. +There are two possible approaches to publishing the index-level +``{name}-{version}-variants.json`` file for every package version: +it can either be prepared and uploaded by the user, or it can be +generated automatically by the index. + +The file should not be changed once it is published, as clients may have +already cached it or locked to the existing hash. For this reason, if +the index is responsible for generating the file, it should use some +mechanism to defer publishing it until the release is fully uploaded +(for example, :pep:`694`). + +To generate the ``{name}-{version}-variants.json`` file: + +1. For the first variant wheel for a given package version, copy the + data from its ``*.dist-info/variant.json`` file. +2. For subsequent wheels, merge the data from their + ``*.dist-info/variant.json`` files into the existing data: + + - disjoint keys of ``variants`` dictionary are merged together + - common keys of these dictionaries must have exactly the same value + - ``default-priorities.namespace`` list can be replaced if the new + value starts with the old value + - ``default-priorities.feature`` and ``default-priorities.value`` + keys can be added if they were not present in the previous + ``default-priorities.namespace`` value + + +Rationale +========= + +Variant wheels use structured `variant properties`_ to express +multidimensional wheel compatibility matrices. For example, it permits +expressing that a single variant requires certain CPU and GPU features +independently. It can express both AND-style dependencies (such as +different CPU instruction sets) and OR-style dependencies (such as +different GPUs supported by a single package). + +The specification does not impose any formal limits on the number of +properties expressed, and specifically accounts for the possibility of +property sets being very long (for example, a long list of GPUs or CPU +extension sets). To avoid wheel filenames becoming very long, the +property lists are stored inside the wheel and mapped to a short label +that is intended to be human-readable. + +To facilitate variant selection while installing from remote index, +the variant metadata is mirrored in a JSON file published on the index. +This enables installers to obtain variant property mapping without +having to fetch individual wheels. + +The variant ordering algorithm has been proposed with the assumption +that variant properties take precedence over Platform compatibility +tags, as they are primarily used to express user preferences. This +accounts for possible divergence of platform tags, e.g. because a CUDA +variant may require a different minimal libc version, in which case the +selection should be driven by the desired CUDA preference rather than +incidental platform tag difference. + +While the provision of variant properties is deferred to a future PEP to +keep the specification easier to comprehend, a baseline assumption is +made that the compatible properties will be provided in specific order +corresponding to their preference, much like Platform compatibility tags +conventionally are. The variant metadata provides the ability to +override this order at package level. However, namespaces are unordered +by design (e.g. we will not decide upfront which GPU vendors take +precedence) and therefore they always need to be ordered by the package +maintainer. + +A concept of null variant is introduced that is distinct from +non-variant wheels to facilitate a transition period. This variant is +always supported by tools implementing this PEP, and takes precedence +over non-variant wheel. It can therefore be used to provide a distinct +fallback for the cases of no other variant being supported and variant +wheels being unsupported altogether. For example, PyTorch could provide +a much smaller null variant that is used when no GPU is supported, and a +fallback non-variant wheel built for the default CUDA version. + + +Backwards Compatibility +======================= + +Variant wheels add an additional `variant label`_ component to the wheel +filename, causing them to be rejected while verifying the filename. This +permits publishing them on an index alongside non-variant wheels, +without risking previous installer versions accidentally installing +them. It was confirmed that the filename validation algorithms in tools +commonly used today reject it: + +- If both the build tag and the variant label are present, the filename + contains too many components. Example: + + .. code-block:: text + + numpy-2.3.2-1-cp313-cp313t-musllinux_1_2_x86_64-x86_64_v3.whl + ^^^^^^^^^^ + +- If only the variant label is present, the Python tag at third position + will be misinterpreted as a build number. Since the build number must + start with a digit and Python tags do not start with digits, + the filename is considered invalid. Example: + + .. code-block:: text + + numpy-2.3.2-cp313-cp313t-musllinux_1_2_x86_64-x86_64_v3.whl + ^^^^^ + +Aside from this explicit incompatibility, the specification makes +minimal and non-intrusive changes to the binary package format. The +`variant metadata`_ is stored in a separate file in the ``.dist-info`` +directory. Tools that are not directly concerned with variants need only +to update their filename verification algorithm (if there is one) and +preserve the contents of said directory. + +If the new `environment markers`_ are used in wheel dependencies, these +wheels will be incompatible with existing tools. For example, upon +meeting these markers in a dependency from an index, pip will backtrack and +use an older dependency version (if possible). This is a general problem +with the design of environment markers, and not specific to wheel +variants. It is possible to work around it by partially evaluating +environment markers at build time, and removing the markers or +dependencies specific to variant wheels from the non-variant wheel. + + +Security Implications +===================== + +The presence of variant wheels may lead to some of the variants being +subject to less scrutiny than others, and as such becoming easier attack +targets. Particularly, once variant wheel support becomes commonplace, +the non-variant wheels for some packages may be only consumed by users +with outdated tools. However, such attacks assume that the package +publishing workflow is already compromised, in which case more plausible +attack vectors are available, for example via modifying compiled +extensions. + + +How to Teach This +================= + +This PEP is oriented at tool authors. Its changes will be integrated +into :doc:`packaging:specifications/binary-distribution-format` and +other PyPA specifications. Teaching variants to end users will be +covered in a subsequent PEP, as user experience details are addressed. + + +Reference Implementation +======================== + +The `variantlib `__ project +contains a reference implementation of a complete variant wheel +solution. It is compliant with this PEP, but also goes beyond it, +providing example solutions to `open issues`_. + +A client for installing variant wheels is implemented in a +`uv branch `__. + + +Rejected Ideas +============== + +Predictable variant labels +-------------------------- + +The specification proposes that variant labels are arbitrary, and +variant properties are mapped to them via a `variant metadata`_ file +rather than expressed directly in them. While it could be technically +possible to create variant labels from variant properties, this would +either require permitting very long filenames that will cause issues +with some platforms, or imposing arbitrary limits on variant property +counts, making the specification less suitable for addressing +multidimensional compatibility matrices. + + +Variant label as part of Platform compatibility tag +--------------------------------------------------- + +The specification adds the variant label as a separate component, +therefore breaking compatibility with existing tools. It could be +technically possible to preserve partial compatibility by appending it +to one of the Platform compatibility tags instead, in which case +installers would reject the wheel based on platform (or Python +interpreter) incompatibility, while other tools could still use it. +However, the authors decided it safer to break the backwards +compatibility. Additionally, reusing tags posed a potential risk of +wheel labels being incorrectly combined with compressed tag sets. For +example, a ``manylinux_2_27_x86_64.manylinux_2_28_x86_64+x8664v3`` tag +would be incorrectly deemed compatible because of the +``manylinux_2_27_x86_64`` part. + + +Open Issues +=========== + +The following problems are deferred to subsequent PEPs: + +- governance of variant namespaces +- determining which variant properties are compatible with the system +- building variant wheels + + +Acknowledgements +================ + +This work would not have been possible without the contributions and +feedback of many people in the Python packaging community. In +particular, we would like to credit the following individuals for their +help in shaping this PEP (in alphabetical order): + +Alban Desmaison, Bradley Dice, Chris Gottbrath, Dmitry Rogozhkin, +Emma Smith, Geoffrey Thomas, Henry Schreiner, Jeff Daily, Jeremy Tanner, +Jithun Nair, Keith Kraus, Leo Fang, Mike McCarty, Nikita Shulga, +Paul Ganssle, Philip Hyunsu Cho, Robert Maynard, Vyas Ramasubramani, +and Zanie Blue. + + +Change History +============== + +- 10-Feb-2026 + + - Initial version, split from :pep:`817` draft. + - Corrected the variant ordering algorithm to order variants per the + best value that is compatible with the system, for every feature, + rather than all compatible values, and add a fallback to ordering + on variant label. + + +Appendices +========== + +- :ref:`9999-variant-json-schema` + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-9999/appendix-core-metadata-json-schema.rst b/peps/pep-9999/appendix-core-metadata-json-schema.rst new file mode 100644 index 00000000000..18870881f7d --- /dev/null +++ b/peps/pep-9999/appendix-core-metadata-json-schema.rst @@ -0,0 +1,11 @@ +:orphan: + +.. _9999-variant-json-schema: + +Appendix: JSON Schema for Variant Metadata +========================================== + +.. literalinclude:: variant_schema.json + :language: json + :linenos: + :name: variant-json-schema diff --git a/peps/pep-9999/variant_schema.json b/peps/pep-9999/variant_schema.json new file mode 100644 index 00000000000..31f45928036 --- /dev/null +++ b/peps/pep-9999/variant_schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://wheelnext.dev/schema/variant-0.0.1.json", + "title": "Variant metadata, v0.0.1", + "description": "The format for variant metadata (variant.json) and index-level metadata ({name}-{version}-variants.json)", + "type": "object", + "properties": { + "default-priorities": { + "description": "Default priorities for ordering variants", + "type": "object", + "properties": { + "namespace": { + "description": "Namespaces (in order of preference)", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9_]+$" + }, + "minItems": 1, + "uniqueItems": true + }, + "feature": { + "description": "Default feature priorities (by namespace)", + "type": "object", + "patternProperties": { + "^[a-z0-9_]+$": { + "description": "The most preferred features", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9_]+$" + }, + "minItems": 0, + "uniqueItems": true + } + }, + "additionalProperties": false, + "uniqueItems": true + }, + "property": { + "description": "Default property priorities (by namespace)", + "type": "object", + "patternProperties": { + "^[a-z0-9_]+$": { + "description": "Default property priorities (by feature name)", + "type": "object", + "patternProperties": { + "^[a-z0-9_]+$": { + "description": "The most preferred feature values", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9_.]+$" + }, + "minItems": 0, + "uniqueItems": true + } + }, + "additionalProperties": false, + "uniqueItems": true + } + }, + "additionalProperties": false, + "uniqueItems": true + } + }, + "additionalProperties": false, + "uniqueItems": true, + "required": [ + "namespace" + ] + }, + "variants": { + "description": "Mapping of variant labels to properties", + "type": "object", + "patternProperties": { + "^[a-z0-9_.]{1,16}$": { + "type": "object", + "description": "Mapping of namespaces in a variant", + "patternProperties": { + "^[a-z0-9_.]+$": { + "description": "Mapping of feature names in a namespace", + "patternProperties": { + "^[a-z0-9_.]+$": { + "description": "List of values for this variant feature", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9_.]+$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "uniqueItems": true, + "additionalProperties": false + } + }, + "uniqueItems": true, + "additionalProperties": false + } + }, + "additionalProperties": false, + "uniqueItems": true + } + }, + "required": [ + "default-priorities", + "variants" + ], + "additionalProperties": false, + "uniqueItems": true +}