diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9ecc51d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 70bc210..0c376df 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: deploy: - description: 'Publish docs?' + description: "Publish docs?" required: true type: boolean @@ -32,6 +32,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@v5 with: @@ -58,8 +59,8 @@ jobs: if: inputs.deploy permissions: - pages: write # to deploy to Pages - id-token: write # to verify the deployment originates from an appropriate source + pages: write # to deploy to Pages + id-token: write # to verify the deployment originates from an appropriate source # Deploy to the github-pages environment environment: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e920e39..23fd00d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..4b51d39 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,20 @@ +name: pre-commit checks + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + name: pre-commit-hooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index b97ecc8..54a1b43 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,4 @@ Thumbs.db # Common editor files *~ *.swp +*.swo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..22fe6c5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,163 @@ +# https://pre-commit.com/ +# +# Before first use: +# +# $ pre-commit install +# +ci: + autofix_prs: false + autoupdate_schedule: quarterly + autoupdate_commit_msg: "chore: update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + skip: [no-commit-to-branch] +fail_fast: false +default_language_version: + python: python3 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + # Sanity checks + - id: check-added-large-files + - id: check-case-conflict + # - id: check-executables-have-shebangs # No executable files yet + - id: check-illegal-windows-names + - id: check-merge-conflict + # Checks based on file type + - id: check-ast + # - id: check-json # No json files yet + - id: check-symlinks + - id: check-toml + # - id: check-xml # No xml files yet + - id: check-yaml + # Detect mistakes + - id: check-vcs-permalinks + - id: debug-statements + - id: destroyed-symlinks + - id: detect-private-key + - id: forbid-submodules + # Automatic fixes + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + # - id: requirements-txt-fixer # No requirements.txt file yet + - id: trailing-whitespace + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.24.1 + hooks: + - id: validate-pyproject + name: Validate pyproject.toml + # Remove unnecessary imports (currently behaves better than ruff) + - repo: https://github.com/PyCQA/autoflake + rev: v2.3.1 + hooks: + - id: autoflake + args: [--in-place] + # Let's keep `pyupgrade` even though `ruff --fix` probably does most of it + - repo: https://github.com/asottile/pyupgrade + rev: v3.20.0 + hooks: + - id: pyupgrade + args: [--py310-plus] + # black often looks better than ruff-format + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 + hooks: + - id: black + - repo: https://github.com/adamchainz/blacken-docs + rev: 1.19.1 + hooks: + - id: blacken-docs + additional_dependencies: [black==25.1.0] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.5 + hooks: + - id: ruff-check + args: [--fix-only, --show-fixes] + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + types_or: [python, markdown, rst, toml, yaml] + additional_dependencies: + - tomli; python_version<'3.11' + - repo: https://github.com/MarcoGorelli/auto-walrus + rev: 0.3.4 + hooks: + - id: auto-walrus + args: [--line-length, "100"] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.5 + hooks: + - id: ruff-check + # - id: ruff-format # Prefer black, but may temporarily uncomment this to see + # `pyroma` may help keep our package standards up to date if best practices change. + # This is probably a "low value" check though and safe to remove if we want faster pre-commit. + # - repo: https://github.com/regebro/pyroma + # rev: "5.0" + # hooks: + # - id: pyroma + # args: [-n, "9", .] # Need author email to score a 10 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.6.2 + hooks: + - id: prettier + args: [--prose-wrap=preserve] + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint + args: [--enable, all, "--disable=line-too-long,leaked-markup"] + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-no-eval + - id: python-no-log-warn + - id: text-unicode-replacement-char + - repo: https://github.com/ComPWA/taplo-pre-commit + rev: v0.9.3 + hooks: + - id: taplo-format + - repo: https://github.com/adrienverge/yamllint + rev: v1.37.1 + hooks: + - id: yamllint + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.33.2 + hooks: + - id: check-dependabot + - id: check-github-workflows + # TODO: get zizmor to pass, and maybe set it up as a github action + # - repo: https://github.com/woodruffw/zizmor-pre-commit + # rev: v1.11.0 + # hooks: + # - id: zizmor + - repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|Github|PyTest|RST|PyLint + exclude: (.pre-commit-config.yaml|docs/pages/guides/style\.md)$ + - id: disallow-words + name: Disallow certain words + language: pygrep + entry: "[Ff]alsey" + exclude: .pre-commit-config.yaml$ + - id: disallow-bad-permalinks + name: Disallow _ in permalinks + language: pygrep + entry: "^permalink:.*_.*" + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: no-commit-to-branch # No commit directly to main + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..fb66779 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,5 @@ +--- +extends: default +rules: + document-start: disable + line-length: disable diff --git a/README.md b/README.md index ac43ba7..e17adc6 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ based on feedback.** Spatch is a dispatching tool with a focus on scientific python libraries. It integrates two forms of dispatching into a single backend system: -* Type dispatching for the main type used by a library. + +- Type dispatching for the main type used by a library. In the scientific python world, this is often the array object. -* Backend selection to offer alternative implementations to users. +- Backend selection to offer alternative implementations to users. These may be faster or less precise, but using them should typically not change code behavior drastically. @@ -26,7 +27,7 @@ to a large scale deployment. Unfortunately, providing code for a host of such types isn't easy and the original library authors usually have neither the bandwidth nor -expertise to do it. Additionally, such layers would have to be optional +expertise to do it. Additionally, such layers would have to be optional components of the library. `spatch` allows for a solution to this dilemma by allowing a third party @@ -35,7 +36,7 @@ to enable library functions to work with alternative types. It should be noted that spatch is not a generic multiple dispatching library. It is opinionated about being strictly typed (we can and probably will support subclasses in the future, though). -It also considers all arguments identically. I.e. if a function takes +It also considers all arguments identically. I.e. if a function takes two inputs (of the kind we dispatch for), there is no distinction for their order. Besides these two things, `spatch` is however a typical type dispatching @@ -56,11 +57,12 @@ For example, we may have a faster algorithm that is parallelized while the old one was not. Or an implementation that dispatches to the GPU but still returns NumPy arrays (as the library always did). -Backend selection _modifies_ behavior rather than extending it. In some +Backend selection _modifies_ behavior rather than extending it. In some cases those modifications may be small (maybe it is really only faster). For the user, backend _selection_ often means that they should explicitly select a preferred backend (e.g. over the default implementation). This could be for example via a context manager: + ```python with backend_opts(prioritize="gpu_backend"): library.function() # now running on the GPU @@ -76,36 +78,38 @@ with backend_opts(prioritize="gpu_backend"): it should be considered a prototype when it comes to API stability. Some examples for missing things we are still working on: -* No way to conveniently see which backends may be used when calling a + +- No way to conveniently see which backends may be used when calling a function (rather than actually calling it). And probably more inspection utilities. -* We have implemented the ability for a backend to defer and not run, +- We have implemented the ability for a backend to defer and not run, but not the ability to run anyway if there is no alternative. -* The main library implementation currently can't distinguish fallback +- The main library implementation currently can't distinguish fallback and default path easily. It should be easy to do this (two functions, `should_run`, or just via `uses_context`). -* `spatch` is very much designed to be fast but that doesn't mean it - is particularly fast yet. We may need to optimize parts (potentially +- `spatch` is very much designed to be fast but that doesn't mean it + is particularly fast yet. We may need to optimize parts (potentially lowering parts to a compiled language). -* We have not implemented tools to test backends, e.g. against parts - of the original library. We expect that "spatch" actually includes most - tools to do this. For example, we could define a `convert` function +- We have not implemented tools to test backends, e.g. against parts + of the original library. We expect that "spatch" actually includes most + tools to do this. For example, we could define a `convert` function that backends can implement to convert arguments in tests as needed. There are also many smaller or bigger open questions and those include whether the API proposed here is actually quite what we want. Other things are for example whether we want API like: -* `dispatchable.invoke(type=, backend=)`. -* Maybe libraries should use `like=` in functions that take no dispatchable + +- `dispatchable.invoke(type=, backend=)`. +- Maybe libraries should use `like=` in functions that take no dispatchable arguments. -* How do we do classes such as scikit-learn estimators. A simple solution might +- How do we do classes such as scikit-learn estimators. A simple solution might a `get_backend(...)` dispatching explicitly once. But we could use more involved schemes, rather remembering the dispatching state of the `.fit()`. We can also see many small conveniences, for example: -* Extract the dispatchable arguments from type annotations. -* Support a magic `Defer` return, rather than the `should_run` call. +- Extract the dispatchable arguments from type annotations. +- Support a magic `Defer` return, rather than the `should_run` call. # Usage examples diff --git a/docs/Makefile b/docs/Makefile index 3c400d4..39250bf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -21,4 +21,3 @@ html: Makefile # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - diff --git a/docs/source/api/for_backends.rst b/docs/source/api/for_backends.rst index 281d1bf..fc59e3f 100644 --- a/docs/source/api/for_backends.rst +++ b/docs/source/api/for_backends.rst @@ -26,7 +26,7 @@ Before writing a backend, you need to think about a few things: by the user. Please check the example linked above. These example entry-points include -code that means running them modifies them in-place if the `@implements` +code that means running them modifies them in-place if the ``@implements`` decorator is used (see next section). Some of the most important things are: @@ -94,7 +94,7 @@ The following fields are supported for each function: - ``function``: The implementation to dispatch to. - ``should_run`` (optional): A function that gets all inputs (and context) - and can decide to defer. Unless you know things will error, try to make sure + and can decide to defer. Unless you know things will error, try to make sure that this function is light-weight. - ``uses_context``: Whether the implementation needs a ``DispatchContext``. - ``additional_docs`` (optional): Brief text to add to the documentation diff --git a/docs/source/api/for_libraries.rst b/docs/source/api/for_libraries.rst index 3e305dd..c74ffc7 100644 --- a/docs/source/api/for_libraries.rst +++ b/docs/source/api/for_libraries.rst @@ -16,4 +16,3 @@ API to create dispatchable functions .. autoclass:: spatch.backend_system.BackendSystem :class-doc-from: init :members: dispatchable, backend_opts - diff --git a/docs/source/api/for_users.rst b/docs/source/api/for_users.rst index 4623849..9e48ddf 100644 --- a/docs/source/api/for_users.rst +++ b/docs/source/api/for_users.rst @@ -18,13 +18,13 @@ Libraries will re-expose all of this functionality under their own API/names. There are currently three global environment variables to modify dispatching behavior at startup time: -* ``_PRIORITIZE``: Comma seperated list of backends. +* ``_PRIORITIZE``: Comma separated list of backends. This is the same as :py:class:`BackendOpts` ``prioritize=`` option. -* ``_BLOCK``: Comma seperated list of backends. +* ``_BLOCK``: Comma separated list of backends. This prevents loading the backends as if they were not installed. No backend code will be executed (its entry-point will not be loaded). -* ``_SET_ORDER``: Comma seperated list of backend orders. - seperated by ``>``. I.e. ``name1>name2,name3>name2`` means that ``name1`` +* ``_SET_ORDER``: Comma separated list of backend orders. + separated by ``>``. I.e. ``name1>name2,name3>name2`` means that ``name1`` and ``name3`` are ordered before ``name2``. This is more fine-grained than the above two and the above two take precedence. Useful to fix relative order of backends without affecting the priority of backends not listed diff --git a/docs/source/conf.py b/docs/source/conf.py index 1fac2e1..3004ac9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,12 +1,11 @@ from __future__ import annotations -import importlib.metadata -from typing import Any +import importlib_metadata project = "spatch" copyright = "2025, Spatch authors" author = "Spatch authors" -version = release = importlib.metadata.version("spatch") +version = release = importlib_metadata.version("spatch") extensions = [ "myst_parser", diff --git a/docs/source/design_choices.md b/docs/source/design_choices.md index 302ed5b..25fe5f9 100644 --- a/docs/source/design_choices.md +++ b/docs/source/design_choices.md @@ -1,33 +1,34 @@ -`spatch` design choices -======================= +# `spatch` design choices This document is designed as a companion to reading the normal API documentation to answer why the API is designed as it is (not detail questions such as naming). Please always remember that `spatch` serves two distinct use-cases: -* Type dispatching, which extending functionality to new types. E.g. without the backend + +- Type dispatching, which extending functionality to new types. E.g. without the backend you only support NumPy, but with the backend you also support CuPy. -* Alternative backend selection where the backend provides an alternative - implementation that is for example faster. It may even use the GPU +- Alternative backend selection where the backend provides an alternative + implementation that is for example faster. It may even use the GPU but the user might still only work with NumPy arrays. A backend can choose to serve one or both of these. Particularly important design considerations are: -* Light-weight import. -* Fast dispatching especially when the user has backends installed but the code + +- Light-weight import. +- Fast dispatching especially when the user has backends installed but the code is not using them. -* No magic by default: Just installing a backend should _not_ change behavior. +- No magic by default: Just installing a backend should _not_ change behavior. (This largely requires backend discipline and coordination with libraries.) -* Adopting `spatch` should be easy for libraries. -* We should aim to make backend authors lives easy but not by making that +- Adopting `spatch` should be easy for libraries. +- We should aim to make backend authors lives easy but not by making that of library authors much harder. % This section really just to keep the first TOC a bit cleaner... -Specific design questions/reasons ---------------------------------- -The following are some specific questions/arguments. The answers may be +## Specific design questions/reasons + +The following are some specific questions/arguments. The answers may be rambling at times :). ```{contents} @@ -38,13 +39,14 @@ rambling at times :). `spatch` uses entry-points like plugin systems and also like NetworkX. Entry-points allow us to do a few things: -* Good user API: If the backend is installed and adds CuPy support, the user + +- Good user API: If the backend is installed and adds CuPy support, the user has to change no code at all. Since the backend is not included neither in the library nor `cupy` entry-points - are the only solution to this. (For type dispatching.) -* Introspection and documentation: Users can introspect available backends + are the only solution to this. (For type dispatching.) +- Introspection and documentation: Users can introspect available backends right away and the list cannot magically change at run-time due to an import. -* We can push many decisions (such as backend priority order) to the startup time. +- We can push many decisions (such as backend priority order) to the startup time. ### Use of identifier strings for everything @@ -63,7 +65,7 @@ This needs to be done with care, but is useful locally (e.g. array creation with no inputs) or just for experiments. If we think of choosing backends, it may come easier to think about it in terms -of a name. For example `skimage` could have a backend `"cucim"` and that +of a name. For example `skimage` could have a backend `"cucim"` and that backend adds `cupy` support. So the question is why shouldn't the user just activate the cucim backend with @@ -72,39 +74,42 @@ its name and that might change behavior of functions to return cupy arrays? We tried to do this and believe wasted a lot of time trying to find a way to square it. `"cucim"` users might in principle want to do any of three things: + 1. Add `cupy` support to `skimage`, i.e. type dispatching for CuPy. 2. Use the GPU for NumPy arrays (i.e. make code faster hopefully without changing behavior significantly). 3. Enforce _unsafe_ use of `cupy` arrays ("zero code change" story) - because point 2. would incure unnecessary and slow host/device copies. + because point 2. would incur unnecessary and slow host/device copies. The first use-case is implicit: The user never has to do anything, it will always just work. But use-cases 2 and 3, both require some explicit opt-in via -`with backend_opts(...)`. We could have distinguished these two with two backend +`with backend_opts(...)`. We could have distinguished these two with two backend names so users would either do `with backend_opts("cucim[numpy]")` or `with backend_opts("cucim[cupy]")`. And that may be easier to understand for users. But unfortunately, it is trying to reduce a two dimensional problem into a one dimensional one and this lead to a long tail of paper-cuts: -* You still need to educate users about `cucim[numpy]` and `cucim[cupy]` and it + +- You still need to educate users about `cucim[numpy]` and `cucim[cupy]` and it is no easier for the backend to implement both. Does `cucim` need 2-3 backends that look similar? Or a mechanism to have multiple - names for one backend? But then the type logic inside the backend depends on the + names for one backend? But then the type logic inside the backend depends on the name in complicated ways, while in `spatch` it depends exactly on `DispatchContext.types`. -* It isn't perfectly clear what happens if `cucim[cupy]` is missing a function. +- It isn't perfectly clear what happens if `cucim[cupy]` is missing a function. Maybe there is another backend to run, but it won't know about the `[cupy]` information! -* If there was a backend that also takes cupy arrays, but has a faster, less-precise +- If there was a backend that also takes cupy arrays, but has a faster, less-precise version, the user would have to activate both `cucim-fast[cupy]` and `cucim[cupy]` to get the desired behavior. We firmly believe that teaching users about `cucim[cupy]` or some backend specific (unsafe) option is not significantly easier than teaching them to use: -* `with backend_opts(prioritize="cucim"):` for the safe first case and -* `with backend_opts(type=cupy.ndarray):` for the unsafe second case. + +- `with backend_opts(prioritize="cucim"):` for the safe first case and +- `with backend_opts(type=cupy.ndarray):` for the unsafe second case. (May also include a `priority="cucim"` as well.) And this is much more explicit about the desire ("use cupy") while avoiding above @@ -136,7 +141,7 @@ fallback (or backend) that uses a custom `library.convert(...)` function and calls the default implementation. If libraries are very interested in this, we should consider extending -`spatch` here. But otherwise, we think it should be backends authors taking the +`spatch` here. But otherwise, we think it should be backends authors taking the lead, although that might end up in extending spatch. ### No generic catch-all implementations? @@ -148,7 +153,7 @@ For example, NumPy has `__array_ufunc__` and because all ufuncs are similar Dask can have a single implementation that always works! If there are well structured use-cases (like the NumPy ufunc one) a library -_can_ choose to explicitly support it: Rather than dispatching for `np.add`, +_can_ choose to explicitly support it: Rather than dispatching for `np.add`, you would dispatch for `np.ufunc()` and pass `np.add` as an argument. In general, we accept that this may be useful and could be a future addition. @@ -165,6 +170,7 @@ The reason for this design is convenience, speed, and simplicity. Matching on types is the only truly fast thing, because it allows a design where the decision of which backend to call is done by: + 1. Fetching the current dispatching state. This is very fast, but unavoidable as we must check user configuration. 2. Figuring out which (unique) types the user passed to do type dispatching. @@ -183,7 +189,7 @@ Caching, type-safety, and no accidentally slow behavior explains why However, we could still ask each backend to provide a `matches(types)` function rather than asking them to list primary and secondary types. -This choice is just for convenience right now. Since we insist on types we might +This choice is just for convenience right now. Since we insist on types we might as well handle the these things inside `spatch`. The reason for "primary" and "secondary" type is to make backends simple if @@ -196,14 +202,16 @@ And if you just want a single type, then ignore the "secondary" one... ### How would I create for example an "Array API" backend? -This is actually not a problem at all. But the backend will have to provide +This is actually not a problem at all. But the backend will have to provide an abstract base class and make sure it is importable in a very light-weight way: + ```python class SupportsArrayAPI(abc.ABC): @classmethod def __subclasshook__(cls, C): return hasattr(C, "__array_nanespace__") ``` + Then you can use `"@mymodule:SupportsArrayAPI"` _almost_ like the normal use. ### Why not use dunder methods (like NumPy, NetworkX, or Array API)? @@ -212,13 +220,14 @@ Magic dunders (`__double_underscore_method__`) are a great way to implement type dispatching (it's also how Python operators work)! But it cannot serve our use-case (and also see advantages of entry-points). The reasons for this is that: -* For NumPy/NetworkX, CuPy is the one that provides the type and thus can attach a + +- For NumPy/NetworkX, CuPy is the one that provides the type and thus can attach a magic dunder to it and provide the implementations for all functions. But for `spatch` the implementation would be a third party (and not e.g. CuPy). And a third party can't easily attach a dunder (e.g. to `cupy.ndarray`) It would not be reliable or work well with the entry-point path to avoid costly - imports. Rather than `spatch` providing the entry-point, cupy would have to. -* Dunders really don't solve the backend selection problem. If they wanted to, + imports. Rather than `spatch` providing the entry-point, cupy would have to. +- Dunders really don't solve the backend selection problem. If they wanted to, you would need another layer pushing the backend selection into types. This may be possible, but would require the whole infrastructure to be centered around the type (i.e. `cupy.ndarray`) rather than the library @@ -228,8 +237,8 @@ We would agree that piggy backing on an existing dunder approach (such as the Array API) seems nice. But ultimately it would be far less flexible (Array API has no backend selection) and be more complex since the Array API would have to provide infrastructure that -can deal with aribtrary libraries different backends for each of them. -To us, it seems simply the wrong way around: The library should dispatch and it +can deal with arbitrary libraries different backends for each of them. +To us, it seems simply the wrong way around: The library should dispatch and it can dispatch to a function that uses the Array API. ### Context manager to modify the dispatching options/state @@ -247,37 +256,42 @@ Context managers simply serve this use-case nicely, quickly, and locally. #### Why not a namespace for explicit dispatching? Based on ideas from NEP 37, the Array API for example dispatches once and then the user has an -`xp` namespace they can pass around. That is great! +`xp` namespace they can pass around. That is great! + +But it is not targeted to end-users. An end-users should write: -But it is not targeted to end-users. An end-users should write: ``` library.function(cupy_array) ``` + and not: + ``` libx = library.dispatch(cupy_array) libx.function(cupy_array) ``` + The second is very explicit and allows to explicitly pass around a "dispatched" state -(i.e. the `libx` namespace). This is can be amazing to write a function that +(i.e. the `libx` namespace). This is can be amazing to write a function that wants to work with different inputs because by using `libx` you don't have to worry about anything and you can even pass it around. -So we love this concept! But we think that the first "end-user" style use-case has to +So we love this concept! But we think that the first "end-user" style use-case has to be center stage for `spatch`. For the core libraries (i.e. NumPy vs. CuPy) the explicit library -use-case seems just far more relevant. We think the reason for this are: -* It is just simpler there, since there no risk of having multiple such contexts. -* Backend selection is not a thing, if it was NumPy or CuPy should naturally handle it +use-case seems just far more relevant. We think the reason for this are: + +- It is just simpler there, since there no risk of having multiple such contexts. +- Backend selection is not a thing, if it was NumPy or CuPy should naturally handle it themselves. (Tis refers to backend selection for speed, not type dispatching. NumPy _does_ type dispatch to cupy and while unsafe, it would not be unfathomable to ask NumPy to dispatch even creation function within a context.) -* User need: It seems much more practical for end-users to just use cupy maybe via +- User need: It seems much more practical for end-users to just use cupy maybe via `import cupy as np` then it is to also modify many library imports. ```{admonition} Future note -`spatch` endevors to provide a more explicit path (and if this gets outdated, maybe we do). +`spatch` endeavors to provide a more explicit path (and if this gets outdated, maybe we do). We expect this to be more of the form of `library.function.invoke(state, ...)` or also `state.invoke(library.function, ...)` or `library.function.invoke(backend=)(...)`. @@ -288,7 +302,7 @@ We do not consider a "dispatched namespace" to be worth the complexity at this p ### The backend priority seems complicated/not complex enough? We need to decide on a priority for backends, which is not an exact science. -For backend-selection there is no hard-and-fast rule: Normally an alternative +For backend-selection there is no hard-and-fast rule: Normally an alternative implementation should have lower priority unless all sides agree it is drop-in enough to be always used. @@ -300,7 +314,7 @@ B accepts, then we must prefer the B because it matches more precisely This is the equivalence to the Python binary operator rule "subclass before superclass". This rule is important, for example because B may be trying to correct behavior of A. -Now accepts "superclass" in the above is a rather broad term. For example backend A +Now accepts "superclass" in the above is a rather broad term. For example backend A may accept `numpy.ndarray | cupy.ndarray` while B only accepts `numpy.ndarray`. "NumPy or CuPy array" here is a superclass of NumPy array. Alternatively, if A accepts all subclasses of `numpy.ndarray` and B accepts only @@ -308,19 +322,20 @@ exactly `numpy.ndarray` (which spatch supports), then A is again the "superclass" because `numpy.ndarray` is also a subclasses of `numpy.ndarray` so A accepts a broader class of inputs. -In practice, the situation is even more complex, though. It is possible that +In practice, the situation is even more complex, though. It is possible that neither backend represents a clear superclass of the other and the above fails to establish an order. So we try to do the above, because we think it simplifies life of backend authors and ensure the correct type-dispatching order in some cases. -But of course it is not perfect! We think it would be a disservice to users to +But of course it is not perfect! We think it would be a disservice to users to attempt a more precise solution (no solution is perfect), because we want to provide users with an understandable, ordered, list of backends, but: -* We can't import all types for correct `issubclass` checks and we don't want the + +- We can't import all types for correct `issubclass` checks and we don't want the priority order to change if types get imported later. -* An extremely precise order might even consider types passed by the user but +- An extremely precise order might even consider types passed by the user but a context dependent order would be too hard to understand. So we infer the correct "type order" where it seems clear/simple enough and otherwise @@ -330,17 +345,17 @@ neither backend authors or end-users need to understand the "how". Of course, one can avoid this complexity by just asking backends to fix the order where if it matters. -We believe that the current complexity in spatch is managable, although we would agree that +We believe that the current complexity in spatch is manageable, although we would agree that a library that isn't dedicated to dispatching should likely avoid it. - ### Choice of environment variables -We don't mind changing these at all. There are three because: -* We need `_BLOCK` to allow avoiding loading a buggy backend entirely. +We don't mind changing these at all. There are three because: + +- We need `_BLOCK` to allow avoiding loading a buggy backend entirely. It is named "block" just because "disable" also makes sense at runtime. -* `_PRIORITIZE` is there as the main user API. -* `_SET_ORDER` is fine grained to prioritize one backend over another. +- `_PRIORITIZE` is there as the main user API. +- `_SET_ORDER` is fine grained to prioritize one backend over another. At runtime this seemed less important (can be done via introspection). This largely exists because if backend ordering is buggy, we tell users to set this to work around the issue. diff --git a/pyproject.toml b/pyproject.toml index fe93614..5593c38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,39 @@ [build-system] -requires = ["hatchling", "hatch-vcs", "hatch-fancy-pypi-readme"] +requires = ["hatchling >=1.26", "hatch-vcs", "hatch-fancy-pypi-readme"] build-backend = "hatchling.build" [project] name = "spatch" -authors = [ - {name = "Scientific Python Developers"}, -] +authors = [{ name = "Scientific Python Developers" }] dynamic = ["version"] -description = "Coming soon" +description = "Coming soon: Python library for enabling dispatching to backends" readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.10" dependencies = ["importlib_metadata"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = ["dispatching"] + +[project.urls] +homepage = "https://github.com/scientific-python/spatch" +documentation = "https://scientific-python.github.io/spatch" +source = "https://github.com/scientific-python/spatch" +changelog = "https://github.com/scientific-python/spatch/releases" [dependency-groups] # black is used to format entry-point files @@ -24,14 +45,15 @@ test = [ { include-group = "backend_utils" }, ] dev = [ + "pre-commit >=4.1", { include-group = "test" }, { include-group = "docs" }, ] docs = [ - "sphinx>=7.0", + "sphinx >=7.0", "sphinx-copybutton", "pydata-sphinx-theme", - "myst-parser", + "myst-parser >=0.13", ] [tool.hatch.version] @@ -48,16 +70,168 @@ backend1 = 'spatch._spatch_example.entry_point' backend2 = 'spatch._spatch_example.entry_point2' [tool.pytest.ini_options] +minversion = "6.0" doctest_plus = "enabled" testpaths = [ - "tests", - "src/spatch", # for doc testing - "docs", + "tests", + "src/spatch", # for doc testing + "docs", ] +xfail_strict = true norecursedirs = ["src"] addopts = [ - "--doctest-glob=docs/source/**.md", + "--doctest-glob=docs/source/**.md", + "--strict-config", # Force error if config is misspelled + "--strict-markers", # Force error if marker is misspelled (must be defined in config) + "-ra", # Print summary of all fails/errors +] +log_cli_level = "info" +filterwarnings = ["error"] + +[tool.black] +line-length = 100 +target-version = ["py310", "py311", "py312", "py313"] + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +extend-select = [ + # Defaults from https://github.com/scientific-python/cookie excluding ARG and EM + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "EXE", # flake8-executable + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "NPY", # NumPy-specific rules + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "T20", # flake8-print + "UP", # pyupgrade + "YTT", # flake8-2020 + + # Additional ones (may be unnecessary, low value, or a nuisance). + # It's okay to experiment and add or remove checks from here + "S", # bandit + "A", # flake8-builtins + "COM", # flake8-commas + "DTZ", # flake8-datetimez + "T10", # flake8-debugger + "ISC", # flake8-implicit-str-concat + "INP", # flake8-no-pep420 + "Q", # flake8-quotes + "RSE", # flake8-raise + "N", # pep8-naming + "PLC", # pylint Convention + "PLE", # pylint Error + "PLR", # pylint Refactor + "PLW", # pylint Warning + "E", # pycodestyle Error + "W", # pycodestyle Warning + "F", # pyflakes + "TRY", # tryceratops + + # Maybe consider + # "EM", # flake8-errmsg (Perhaps nicer, but it's a lot of work) + # "ALL", # to see everything! +] +unfixable = [ + "F841", # unused-variable (Note: can leave useless expression) + "B905", # zip-without-explicit-strict (Note: prefer `zip(x, y, strict=True)`) ] +ignore = [ + # Maybe consider + "ANN", # flake8-annotations (We don't fully use annotations yet) + "B904", # Use `raise from` to specify exception cause (Note: sometimes okay to raise original exception) + "PERF401", # Use a list comprehension to create a transformed list (Note: poorly implemented atm) + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` (Note: no annotations yet) + "RUF021", # parenthesize-chained-operators (Note: results don't look good yet) + "RUF023", # unsorted-dunder-slots (Note: maybe fine, but noisy changes) + "S310", # Audit URL open for permitted schemes (Note: we don't download URLs in normal usage) + "TRY004", # Prefer `TypeError` exception for invalid type (Note: good advice, but not worth the nuisance) + "TRY301", # Abstract `raise` to an inner function + + # Intentionally ignored + "B019", # Use of `functools.lru_cache` or `functools.cache` on methods can lead to memory leaks + "COM812", # Trailing comma missing + "D203", # 1 blank line required before class docstring (Note: conflicts with D211, which is preferred) + "D213", # (Note: conflicts with D212, which is preferred) + "D400", # First line should end with a period (Note: prefer D415, which also allows "?" and "!") + "N801", # Class name ... should use CapWords convention (Note:we have a few exceptions to this) + "N802", # Function name ... should be lowercase + "N803", # Argument name ... should be lowercase (Maybe okay--except in tests) + "N806", # Variable ... in function should be lowercase + "N807", # Function name should not start and end with `__` + "N818", # Exception name ... should be named with an Error suffix (Note: good advice) + "PERF203", # `try`-`except` within a loop incurs performance overhead (Note: too strict) + "PLC0205", # Class `__slots__` should be a non-string iterable (Note: string is fine) + "PLC0415", # `import` should be at the top-level of a file (Note: good advice, too strict) + "PLR0124", # Name compared with itself, consider replacing `x == x` (Note: too strict) + "PLR0911", # Too many return statements + "PLR0912", # Too many branches + "PLR0913", # Too many arguments to function call + "PLR0915", # Too many statements + "PLR2004", # Magic number used in comparison, consider replacing magic with a constant variable + "PLW0603", # Using the global statement to update ... is discouraged (Note: yeah, discouraged, but too strict) + "PLW0642", # Reassigned `self` variable in instance method (Note: too strict for us) + "PLW2901", # Outer for loop variable ... overwritten by inner assignment target (Note: good advice, but too strict) + "RET502", # Do not implicitly `return None` in function able to return non-`None` value + "RET503", # Missing explicit `return` at the end of function able to return non-`None` value + "RET504", # Unnecessary variable assignment before `return` statement + "RUF018", # Avoid assignment expressions in `assert` statements + "S110", # `try`-`except`-`pass` detected, consider logging the exception (Note: good advice, but we don't log) + "S112", # `try`-`except`-`continue` detected, consider logging the exception (Note: good advice, but we don't log) + "S603", # `subprocess` call: check for execution of untrusted input (Note: not important for us) + "S607", # Starting a process with a partial executable path (Note: not important for us) + "SIM102", # Use a single `if` statement instead of nested `if` statements (Note: often necessary) + "SIM105", # Use contextlib.suppress(...) instead of try-except-pass (Note: try-except-pass is much faster) + "SIM108", # Use ternary operator ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) + "TID251", # flake8-tidy-imports.banned-api + "TRY003", # Avoid specifying long messages outside the exception class (Note: why?) + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` (Note: using `|` is slower atm) + + # Ignored categories + "C90", # mccabe (Too strict, but maybe we should make things less complex) + "BLE", # flake8-blind-except (Maybe consider) + "FBT", # flake8-boolean-trap (Why?) + "DJ", # flake8-django (We don't use django) + "PYI", # flake8-pyi (We don't have stub files yet) + "SLF", # flake8-self (We can use our own private variables--sheesh!) + "TCH", # flake8-type-checking (Note: figure out type checking later) + "ARG", # flake8-unused-arguments (Sometimes helpful, but too strict) + "TD", # flake8-todos (Maybe okay to add some of these) + "FIX", # flake8-fixme (like flake8-todos) + "ERA", # eradicate (We like code in comments!) + "PD", # pandas-vet (Intended for scripts that use pandas, not libraries) +] + +[tool.ruff.lint.per-file-ignores] +"src/spatch/**/__init__.py" = [ + "F401", # Allow unused import (w/o defining `__all__`) +] +"src/spatch/_spatch_example/backend.py" = ["T201"] # Allow print +"docs/**/*.py" = ["INP001"] # Not a package +"tests/*.py" = [ + "S101", # Allow assert + "INP001", # Not a package +] + +[tool.ruff.lint.flake8-builtins] +builtins-ignorelist = ["copyright", "type"] + +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false + +[tool.ruff.lint.pydocstyle] +convention = "numpy" [tool.coverage] run.source = ["spatch"] diff --git a/src/spatch/_spatch_example/README.md b/src/spatch/_spatch_example/README.md index 0401624..68429c1 100644 --- a/src/spatch/_spatch_example/README.md +++ b/src/spatch/_spatch_example/README.md @@ -1,6 +1,6 @@ # Minimal usage example -A minimal example can be found below. If you are interested in the +A minimal example can be found below. If you are interested in the implementation side of this, please check [the source](https://github.com/scientific-python/spatch/spatch/_spatch_example). @@ -8,17 +8,19 @@ This is a very minimalistic example to show some of the concepts of creating a library and backends and how a user might work with them. The "library" contains only: -* `backend_opts()` a context manager for the user to change dispatching. -* A `divide` function that is dispatching enabled and assumed to be + +- `backend_opts()` a context manager for the user to change dispatching. +- A `divide` function that is dispatching enabled and assumed to be designed for only `int` inputs. We then have two backends with their corresponding definitions in `backend.py`. The entry-points are `entry_point.py` and `entry_point2.py` and these files can be run to generate their `functions` context (i.e. if you add more functions). -For users we have the following basic capabilities. Starting with normal +For users we have the following basic capabilities. Starting with normal type dispatching. First, import the functions and set up tracing globally: + ```pycon >>> import pprint >>> from spatch._spatch_example.library import divide, backend_opts @@ -26,14 +28,16 @@ First, import the functions and set up tracing globally: >>> opts.enable_globally() # or with opts() as trace: ``` + Now try calling the various implementations: + ```pycon >>> divide(1, 2) # use the normal library implementation (int inputs) 0 ->>> divide(1., 2.) # uses backend 1 (float input) +>>> divide(1.0, 2.0) # uses backend 1 (float input) hello from backend 1 0.5 ->>> divide(1j, 2.) # uses backend 2 (complex input) +>>> divide(1j, 2.0) # uses backend 2 (complex input) hello from backend 2 0.5j >>> pprint.pprint(opts.trace) @@ -50,18 +54,23 @@ The first thing is to prioritize the use of a backend over another Backend 1 also has integers as a primary type, so we can prefer it over the default implementation for integer inputs as well: + ```pycon >>> with backend_opts(prioritize="backend1"): ... divide(1, 2) # now uses backend1 +... hello from backend 1 0 ``` + Similarly backend 2 supports floats, so we can prefer it over backend 1. We can still also prioritize "backend1" if we want: + ```pycon >>> with backend_opts(prioritize=["backend2", "backend1"]): -... divide(1., 2.) # now uses backend2 +... divide(1.0, 2.0) # now uses backend2 +... hello from backend 2 0.5 >>> pprint.pprint(opts.trace[-2:]) @@ -69,39 +78,45 @@ hello from backend 2 ('spatch._spatch_example.library:divide', [('backend2', 'called')])] ``` + The default priorities are based on the backend types or an explicit request to have a higher priority by a backend (otherwise default first and then alphabetically). Backends do have to make sure that the priorities make sense (i.e. there are no priority circles). -Prioritizing a backend will often effectively enable it. If such a backend changes +Prioritizing a backend will often effectively enable it. If such a backend changes behavior (e.g. faster but less precision) this can change results and confuse third party library functions. This is a user worry, backends must make sure that they never change types (even if prioritized), though. In the array world there use-cases that are not covered in the above: -* There are functions that create new arrays (say random number generators) - without inputs. We may wish to change their behavior within a scope or + +- There are functions that create new arrays (say random number generators) + without inputs. We may wish to change their behavior within a scope or globally. -* A user may try to bluntly modify behavior to use e.g. arrays on the GPU. +- A user may try to bluntly modify behavior to use e.g. arrays on the GPU. This is supported, but requires indicating the _type_ preference and users must be aware that this can even easier break their or third party code: + ```pycon >>> with backend_opts(type=float): ... divide(1, 2) # returns float (via backend 1) +... hello from backend 1 0.5 >>> with backend_opts(type=complex): ... # backen 2 returning a float for complex "input" is probably OK -... # (but may be debateable) +... # (but may be debatable) ... divide(1, 2) +... hello from backend 2 0.5 >>> with backend_opts(type=float, prioritize="backend2"): -... # we can of course combine both type and prioritize. -... divide(1, 2) # backend 2 with float result (int inputs). +... # we can of course combine both type and prioritize. +... divide(1, 2) # backend 2 with float result (int inputs). +... hello from backend 2 0.5 >>> pprint.pprint(opts.trace[-3:]) @@ -110,7 +125,8 @@ hello from backend 2 ('spatch._spatch_example.library:divide', [('backend2', 'called')])] ``` + How types work precisely should be decided by the backend, but care should be taken. E.g. it is not clear if returning a float is OK when the user said `type=complex`. (In the future, we may want to think more about this, especially if `type=complex|real` -may make sense, or if we should fall back if no implementation can be found.) \ No newline at end of file +may make sense, or if we should fall back if no implementation can be found.) diff --git a/src/spatch/_spatch_example/backend.py b/src/spatch/_spatch_example/backend.py index cb0f307..55d14fc 100644 --- a/src/spatch/_spatch_example/backend.py +++ b/src/spatch/_spatch_example/backend.py @@ -1,6 +1,7 @@ try: from spatch.backend_utils import BackendImplementation except ModuleNotFoundError: # pragma: no cover + class Noop: # No-operation/do nothing version of a BackendImplementation def __call__(self, *args, **kwargs): @@ -15,9 +16,9 @@ def __call__(self, *args, **kwargs): backend1 = BackendImplementation("backend1") backend2 = BackendImplementation("backend2") + # For backend 1 -@backend1.implements( - library.divide, uses_context=True, should_run=lambda info, x, y: True) +@backend1.implements(library.divide, uses_context=True, should_run=lambda info, x, y: True) def divide(context, x, y): """This implementation works well on floats.""" print("hello from backend 1") diff --git a/src/spatch/_spatch_example/entry_point.py b/src/spatch/_spatch_example/entry_point.py index b0906bd..791e021 100644 --- a/src/spatch/_spatch_example/entry_point.py +++ b/src/spatch/_spatch_example/entry_point.py @@ -24,7 +24,7 @@ if __name__ == "__main__": # pragma: no cover # Run this file as a script to update this file - from spatch.backend_utils import update_entrypoint from spatch._spatch_example.backend import backend1 + from spatch.backend_utils import update_entrypoint update_entrypoint(__file__, backend1, "spatch._spatch_example.backend") diff --git a/src/spatch/_spatch_example/entry_point2.py b/src/spatch/_spatch_example/entry_point2.py index 5831b7f..9355d64 100644 --- a/src/spatch/_spatch_example/entry_point2.py +++ b/src/spatch/_spatch_example/entry_point2.py @@ -19,7 +19,7 @@ if __name__ == "__main__": # pragma: no cover # Run this file as a script to update this file - from spatch.backend_utils import update_entrypoint from spatch._spatch_example.backend import backend2 + from spatch.backend_utils import update_entrypoint update_entrypoint(__file__, backend2, "spatch._spatch_example.backend") diff --git a/src/spatch/_spatch_example/library.py b/src/spatch/_spatch_example/library.py index 5b05fec..66e7027 100644 --- a/src/spatch/_spatch_example/library.py +++ b/src/spatch/_spatch_example/library.py @@ -1,15 +1,15 @@ - from spatch.backend_system import BackendSystem _backend_system = BackendSystem( "_spatch_example_backends", # entry point group "_SPATCH_EXAMPLE_BACKENDS", # environment variable prefix - default_primary_types=["builtins:int"] + default_primary_types=["builtins:int"], ) backend_opts = _backend_system.backend_opts + @_backend_system.dispatchable(["x", "y"]) def divide(x, y): """Divide integers, other types may be supported via backends""" diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 33e0b89..a98b8f9 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -1,19 +1,19 @@ -import contextlib import contextvars import dataclasses -from dataclasses import dataclass import functools import os -import importlib_metadata -import warnings import sys import textwrap +import warnings +from collections.abc import Callable +from dataclasses import dataclass from types import MethodType -from typing import Any, Callable +from typing import Any -from spatch import from_identifier, get_identifier -from spatch.utils import TypeIdentifier, valid_backend_name +import importlib_metadata +from spatch import from_identifier, get_identifier +from spatch.utils import EMPTY_TYPE_IDENTIFIER, TypeIdentifier, valid_backend_name __doctest_skip__ = ["BackendOpts.__init__"] @@ -21,9 +21,9 @@ @dataclass(slots=True) class Backend: name: str - primary_types: TypeIdentifier = TypeIdentifier([]) - secondary_types: TypeIdentifier = TypeIdentifier([]) - functions : dict = dataclasses.field(default_factory=dict) + primary_types: TypeIdentifier = EMPTY_TYPE_IDENTIFIER + secondary_types: TypeIdentifier = EMPTY_TYPE_IDENTIFIER + functions: dict = dataclasses.field(default_factory=dict) known_backends: frozenset = frozenset() higher_priority_than: frozenset = frozenset() lower_priority_than: frozenset = frozenset() @@ -54,22 +54,19 @@ def from_namespace(cls, info): def known_type(self, dispatch_type): if dispatch_type in self.primary_types: return "primary" # TODO: maybe make it an enum? - elif dispatch_type in self.secondary_types: + if dispatch_type in self.secondary_types: return "secondary" - else: - return False + return False def matches(self, dispatch_types): matches = frozenset(self.known_type(t) for t in dispatch_types) - if "primary" in matches and False not in matches: - return True - return False + return "primary" in matches and False not in matches def compare_with_other(self, other): # NOTE: This function is a symmetric comparison if other.name in self.higher_priority_than: return 2 - elif other.name in self.lower_priority_than: + if other.name in self.lower_priority_than: return -2 # If our primary types are a subset of the other, we match more @@ -87,30 +84,30 @@ def compare_backends(backend1, backend2, prioritize_over): # Environment variable prioritization beats everything: if (prio := prioritize_over.get(backend1.name)) and backend2.name in prio: return 3 - elif (prio := prioritize_over.get(backend2.name)) and backend1.name in prio: + if (prio := prioritize_over.get(backend2.name)) and backend1.name in prio: return -3 # Sort by the backends compare function (i.e. type hierarchy and manual order). # We default to a type based comparisons but allow overriding this, so check - # both ways (to find the overriding). This also find inconcistencies. + # both ways (to find the overriding). This also find inconsistencies. cmp1 = backend1.compare_with_other(backend2) cmp2 = backend2.compare_with_other(backend1) if cmp1 is NotImplemented and cmp2 is NotImplemented: return 0 - elif cmp1 is NotImplemented: + if cmp1 is NotImplemented: return -cmp2 - elif cmp2 is NotImplemented: + if cmp2 is NotImplemented: return cmp1 if cmp1 == cmp2: raise RuntimeError( "Backends {backend1.name} and {backend2.name} report inconsistent " "priorities (this means they are buggy). You can manually set " - "a priority or remove one of the backends.") + "a priority or remove one of the backends." + ) if cmp1 > cmp2: return cmp1 - else: - return -cmp2 + return -cmp2 def _modified_state( @@ -137,17 +134,16 @@ def _modified_state( if b not in backend_system.backends: if unknown_backends == "raise": raise ValueError(f"Backend '{b}' not found.") - elif unknown_backends == "ignore": + if unknown_backends == "ignore": pass else: - raise ValueError( - "_modified_state() unknown_backends must be raise or ignore") + raise ValueError("_modified_state() unknown_backends must be raise or ignore") - if type is not None: - if not backend_system.known_type(type, primary=True): - raise ValueError( - f"Type '{type}' not a valid primary type of any backend. " - "It is impossible to enforce use of this type for any function.") + if type is not None and not backend_system.known_type(type, primary=True): + raise ValueError( + f"Type '{type}' not a valid primary type of any backend. " + "It is impossible to enforce use of this type for any function." + ) ordered_backends, _, prioritized, curr_trace = curr_state prioritized = prioritized | frozenset(prioritize) @@ -233,7 +229,7 @@ def __init__(self, *, prioritize=(), disable=(), type=None, trace=False): type The type to dispatch for within this context. trace - The trace object (currenly a list as described in the examples). + The trace object (currently a list as described in the examples). If used, the trace is also returned when entering the context. Notes @@ -279,7 +275,7 @@ def __init__(self, *, prioritize=(), disable=(), type=None, trace=False): Backends should simply document their behavior with ``backend_opts`` and which usage pattern they see for their users. - Tracing calls can be done using, where ``trace`` is a list of informations for + Tracing calls can be done using, where ``trace`` is a list of information for each call. This contains a tuple of the function identifier and a list of backends called (typically exactly one, but it will also note if a backend deferred via ``should_run``). @@ -291,7 +287,11 @@ def __init__(self, *, prioritize=(), disable=(), type=None, trace=False): self._state = _modified_state( self._backend_system, self._dispatch_state.get(), - prioritize=prioritize, disable=disable, type=type, trace=trace) + prioritize=prioritize, + disable=disable, + type=type, + trace=trace, + ) # unpack new state to provide information: self.backends, self.prioritized, self.type, self.trace = self._state self._token = None @@ -300,13 +300,14 @@ def __repr__(self): # We could allow a repr that can be copy pasted, but this seems more clear? inactive = tuple(b for b in self._backend_system.backends if b not in self.backends) type_str = " type: {tuple(self.type)[0]!r}\n" if self.type else "" - return (f"" - ) + return ( + f"" + ) def enable_globally(self): """Apply these backend options globally. @@ -317,18 +318,19 @@ def enable_globally(self): is safer to use the contextmanager ``with`` statement instead. This method will issue a warning if the - dispatching state has been previously modified programatically. + dispatching state has been previously modified programmatically. """ curr_state = self._dispatch_state.get(None) # None used before default # If the state was never set or the state matches (ignoring trace) # and there was no trace registered before this is OK. Otherwise warn. if curr_state is not None and ( - curr_state[:-1] != self._state[:-1] - or curr_state[-1] is not None): + curr_state[:-1] != self._state[:-1] or curr_state[-1] is not None + ): warnings.warn( "Backend options were previously modified, global change of the " "backends state should only be done once from the main program.", - UserWarning, 2 + UserWarning, + 2, ) self._token = self._dispatch_state.set(self._state) @@ -374,6 +376,7 @@ def func(): func : callable The decorated function. """ + # In this form, allow entering multiple times by storing the token # inside the wrapper functions locals @functools.wraps(func) @@ -427,8 +430,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N self.backends = {} if default_primary_types is not None: self.backends["default"] = Backend( - name="default", - primary_types=TypeIdentifier(default_primary_types) + name="default", primary_types=TypeIdentifier(default_primary_types) ) try: @@ -442,12 +444,14 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N raise ValueError( f"Invalid order with duplicate backend in environment " f"variable {environ_prefix}_SET_ORDER:\n" - f" {orders_str}") + f" {orders_str}" + ) prev_b = None for b in orders: if not valid_backend_name(b): raise ValueError( - f"Name {b!r} in {environ_prefix}_SET_ORDER is not a valid backend name.") + f"Name {b!r} in {environ_prefix}_SET_ORDER is not a valid backend name." + ) if prev_b is not None: prioritize_over.setdefault(prev_b, set()).add(b) # If an opposite prioritization was already set, discard it. @@ -459,7 +463,8 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N warnings.warn( f"Ignoring invalid environment variable {environ_prefix}_SET_ORDER " f"due to error: {e}", - UserWarning, 2 + UserWarning, + 2, ) try: @@ -468,12 +473,14 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N for b in prioritize: if not valid_backend_name(b): raise ValueError( - f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name.") + f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name." + ) except Exception as e: warnings.warn( f"Ignoring invalid environment variable {environ_prefix}_PRIORITIZE " f"due to error: {e}", - UserWarning, 2 + UserWarning, + 2, ) try: @@ -482,13 +489,16 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N for b in blocked: if not b.isidentifier(): raise ValueError( - f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name.") + f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name." + ) except Exception as e: warnings.warn( f"Ignoring invalid environment variable {environ_prefix}_SET_ORDER " f"due to error: {e}", - UserWarning, 2 + UserWarning, + 2, ) + self._environ_prefix = environ_prefix # Note that the order of adding backends matters, we add `backends` first # and then entry point ones in alphabetical order. @@ -500,10 +510,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N try: self.backend_from_namespace(backend) except Exception as e: - warnings.warn( - f"Skipping backend {backend.name} due to error: {e}", - UserWarning, 2 - ) + warnings.warn(f"Skipping backend {backend.name} due to error: {e}", UserWarning, 2) # Create a directed graph for which backends have a known higher priority than others. # The topological sorter is stable with respect to the original order, so we add @@ -515,7 +522,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N backends = [self.backends[n] for n in graph] for i, b1 in enumerate(backends): - for b2 in backends[i+1:]: + for b2 in backends[i + 1 :]: cmp = compare_backends(b1, b2, prioritize_over) if cmp < 0: graph[b1.name].append(b2.name) @@ -526,32 +533,33 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N # Finalize backends to be a dict sorted by priority. self.backends = {b: self.backends[b] for b in order} - # The state is the ordered (active) backends and the prefered type (None) + # The state is the ordered (active) backends and the preferred type (None) # and the trace (None as not tracing). base_state = (order, None, frozenset(), None) disable = {b.name for b in self.backends.values() if b.requires_opt_in} state = _modified_state( - self, base_state, prioritize=prioritize, disable=disable, unknown_backends="ignore") - self._dispatch_state = contextvars.ContextVar( - f"{group}.dispatch_state", default=state) + self, base_state, prioritize=prioritize, disable=disable, unknown_backends="ignore" + ) + self._dispatch_state = contextvars.ContextVar(f"{group}.dispatch_state", default=state) - @staticmethod - def _toposort(graph): + def _toposort(self, graph): # Adapted from Wikipedia's depth-first pseudocode. We are not using graphlib, # because it doesn't preserve the original order correctly. # This depth-first approach does preserve it. - def visit(node, order, _visiting={}): + def visit(node, order, _visiting): if node in order: return if node in _visiting: - cycle = (tuple(_visiting.keys()) + (node,))[::-1] + cycle = (*_visiting.keys(), node)[::-1] raise RuntimeError( f"Backends form a priority cycle. This is a bug in a backend or your\n" - f"environment settings. Check the environment variable {environ_prefix}_SET_ORDER\n" + "environment settings. Check the environment variable " + f"{self._environ_prefix}_SET_ORDER\n" f"and change it for example to:\n" - f" {environ_prefix}_SET_ORDER=\"{cycle[-1]}>{cycle[-2]}\"\n" + f' {self._environ_prefix}_SET_ORDER="{cycle[-1]}>{cycle[-2]}"\n' f"to break the offending cycle:\n" - f" {'>'.join(cycle)}") from None + f" {'>'.join(cycle)}" + ) from None _visiting[node] = None # mark as visiting/in-progress for n in graph[node]: @@ -560,10 +568,9 @@ def visit(node, order, _visiting={}): del _visiting[node] order[node] = None # add sorted node - to_sort = list(graph.keys()) order = {} # dict as a sorted set - for n in list(graph.keys()): - visit(n, order) + for n in graph: + visit(n, order, {}) return tuple(order.keys()) @@ -584,22 +591,17 @@ def _get_entry_points(group, blocked): namespace = ep.load() if ep.name != namespace.name: raise RuntimeError( - f"Entrypoint name {ep.name!r} and actual name {namespace.name!r} mismatch.") + f"Entrypoint name {ep.name!r} and actual name {namespace.name!r} mismatch." + ) backends.append(namespace) except Exception as e: - warnings.warn( - f"Skipping backend {ep.name} due to error: {e}", - UserWarning, 3 - ) + warnings.warn(f"Skipping backend {ep.name} due to error: {e}", UserWarning, 3) return sorted(backends, key=lambda x: x.name) @functools.lru_cache(maxsize=128) def known_type(self, dispatch_type, primary=False): - for backend in self.backends.values(): - if backend.known_type(dispatch_type): - return True - return False + return any(backend.known_type(dispatch_type) for backend in self.backends.values()) def get_known_unique_types(self, dispatch_types): # From a list of args, return only the set of dispatch types @@ -636,7 +638,8 @@ def backend_from_namespace(self, info_namespace): if new_backend.name in self.backends: warnings.warn( f"Backend of name '{new_backend.name}' already exists. Ignoring second!", - UserWarning, 3 + UserWarning, + 3, ) return self.backends[new_backend.name] = new_backend @@ -681,6 +684,7 @@ def dispatchable(self, dispatch_args=None, *, module=None, qualname=None): Unfortunately, changing the module can confuse some tools, so we may wish to change the behavior of actually overriding it. """ + def wrap_callable(func): # Overwrite original module (we use it later, could also pass it) if module is not None: @@ -698,9 +702,9 @@ def wrap_callable(func): def backend_opts(self): """Property returning a :py:class:`BackendOpts` class specific to this library (tied to this backend system). - """ + """ return type( - f"BackendOpts", + "BackendOpts", (BackendOpts,), {"_dispatch_state": self._dispatch_state, "_backend_system": self}, ) @@ -749,6 +753,7 @@ class DispatchContext: to use this to decide that e.g. a NumPy array will be converted to a cupy array, but only if prioritized. """ + # The idea is for the context to be very light-weight so that specific # information should be properties (because most likely we will never need it). # This object can grow to provide more information to backends. @@ -779,10 +784,9 @@ def function(self): _function = self._function if type(_function) is not str: return _function - else: - _function = from_identifier(_function) - self._function = _function - return _function + _function = from_identifier(_function) + self._function = _function + return _function class _Implentations(dict): @@ -842,17 +846,15 @@ def __init__(self, backend_system, func, dispatch_args, ident=None): for i, p in enumerate(sig.parameters.values()): if p.name not in dispatch_args: continue - if ( - p.kind == inspect.Parameter.POSITIONAL_ONLY or - p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD - ): + if p.kind in {p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD}: # Accepting it as a keyword is irrelevant here (fails later) new_dispatch_args[p.name] = i - elif p.kind == inspect.Parameter.KEYWORD_ONLY: + elif p.kind == p.KEYWORD_ONLY: new_dispatch_args[p.name] = sys.maxsize else: raise TypeError( - f"Parameter {p.name} is variable. Must use callable `dispatch_args`.") + f"Parameter {p.name} is variable. Must use callable `dispatch_args`." + ) if len(dispatch_args) != len(new_dispatch_args): not_found = set(dispatch_args) - set(new_dispatch_args) @@ -887,7 +889,10 @@ def __init__(self, backend_system, func, dispatch_args, ident=None): # Create implementations, lazy loads should_run (and maybe more in the future). self._implementations = _Implentations(impl_infos) self._implementations["default"] = _Implementation( - "default", self._default_func, None, False, + "default", + self._default_func, + None, + False, ) if not new_doc: @@ -911,16 +916,16 @@ def _get_dispatch_args(self, *args, **kwargs): # Return all dispatch args if self._dispatch_args is None: return args + tuple(kwargs.values()) - else: - return tuple( - val for name, pos in self._dispatch_args.items() - if (val := args[pos] if pos < len(args) else kwargs.get(name)) is not None - ) + return tuple( + val + for name, pos in self._dispatch_args.items() + if (val := args[pos] if pos < len(args) else kwargs.get(name)) is not None + ) def __call__(self, *args, **kwargs): dispatch_args = tuple(self._get_dispatch_args(*args, **kwargs)) # At this point dispatch_types is not filtered for known types. - dispatch_types = set(type(val) for val in dispatch_args) + dispatch_types = {type(val) for val in dispatch_args} state = self._backend_system._dispatch_state.get() ordered_backends, type_, prioritized, trace = state @@ -929,7 +934,8 @@ def __call__(self, *args, **kwargs): dispatch_types = frozenset(dispatch_types) dispatch_types, matching_backends = self._backend_system.get_types_and_backends( - dispatch_types, ordered_backends) + dispatch_types, ordered_backends + ) if trace is not None: call_trace = [] @@ -953,20 +959,19 @@ def __call__(self, *args, **kwargs): if impl.uses_context: return impl.function(context, *args, **kwargs) - else: - return impl.function(*args, **kwargs) + return impl.function(*args, **kwargs) - elif should_run is not False: + if should_run is not False: # Strict to allow future use as "should run if needed only". That would merge # "can" and "should" run. I can see a dedicated `can_run`, but see it as more - # useful if `can_run` was passed only cachable parameters (e.g. `method="meth"`, + # useful if `can_run` was passed only cacheable parameters (e.g. `method="meth"`, # or even `backend=`, although that would be special). # (We may tag on a reason for a non-True return value as well or use context.) - raise NotImplementedError(f"Currently, should run must return True or False.") - elif trace is not None and impl.should_run is not None: + raise NotImplementedError("Currently, should run must return True or False.") + if trace is not None and impl.should_run is not None: call_trace.append((name, "skipped due to should_run returning False")) if call_trace is not None: call_trace.append(("default fallback", "called")) - return self._default_func(*args, **kwargs) \ No newline at end of file + return self._default_func(*args, **kwargs) diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index 635566d..4820d14 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -1,5 +1,6 @@ -from dataclasses import dataclass from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path from .utils import from_identifier, get_identifier @@ -20,8 +21,7 @@ class BackendImplementation: impl_to_info: dict[str, BackendFunctionInfo] def __init__(self, backend_name: str): - """Helper class to create backends. - """ + """Helper class to create backends.""" self.name = backend_name self.api_to_info = {} # {api_identity_string: backend_function_info} self.impl_to_info = {} # {impl_identity_string: backend_function_info} @@ -120,7 +120,7 @@ def set_should_run(self, backend_func: str | Callable): def inner(func: Callable): identity = get_identifier(func) - if identity.endswith(":") or identity.endswith(":_"): + if identity.endswith((":", ":_")): backend_func._should_run = func identity = f"{impl_identity}._should_run" info = self.impl_to_info[impl_identity] @@ -163,7 +163,7 @@ def __repr__(self): return "\n".join( [ "(", - *(repr(line + '\n') for line in self.lines[:-1]), + *(repr(line + "\n") for line in self.lines[:-1]), repr(self.lines[-1]), ")", ] @@ -235,8 +235,6 @@ def update_entrypoint( ] # Step 4: replace text - with open(filepath) as f: - text = f.read() + text = Path(filepath).read_text() text = update_text(text, lines, "functions", indent=indent) - with open(filepath, "w") as f: - f.write(text) + Path(filepath).write_text(text) diff --git a/src/spatch/testing.py b/src/spatch/testing.py index de75a75..ef42a78 100644 --- a/src/spatch/testing.py +++ b/src/spatch/testing.py @@ -1,5 +1,6 @@ from spatch.utils import get_identifier + class _FuncGetter: def __init__(self, get): self.get = get @@ -11,6 +12,7 @@ class BackendDummy: Forwards any lookup to the class. Documentation are used from the function which must match in the name. """ + def __init__(self): self.functions = _FuncGetter(self.get_function) @@ -20,7 +22,7 @@ def get_function(cls, name, default=None): _, name = name.split(":") # Not get_identifier because it would find the super-class name. - res = {"function": f"{cls.__module__}:{cls.__name__}.{name}" } + res = {"function": f"{cls.__module__}:{cls.__name__}.{name}"} if hasattr(cls, "uses_context"): res["uses_context"] = cls.uses_context if hasattr(cls, "should_run"): @@ -36,4 +38,3 @@ def get_function(cls, name, default=None): def dummy_func(cls, *args, **kwargs): # Always define a small function that mainly forwards. return cls.name, args, kwargs - diff --git a/src/spatch/utils.py b/src/spatch/utils.py index c82a25d..cf0b916 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -1,9 +1,10 @@ -from importlib import import_module -from importlib.metadata import version -from dataclasses import dataclass, field import re import sys import warnings +from dataclasses import dataclass, field +from importlib import import_module + +from importlib_metadata import version def get_identifier(obj): @@ -20,12 +21,12 @@ def from_identifier(ident): def get_project_version(project_name, *, action_if_not_found="warn", default=None): - """Get the version of a project from ``importlib.metadata``. + """Get the version of a project from ``importlib_metadata``. This is useful to ensure a package is properly installed regardless of the tools used to build the project and create the version. Proper installation is important to ensure entry-points of the project are discoverable. If the - project is not found by ``importlib.metadata``, behavior is controlled by + project is not found by ``importlib_metadata``, behavior is controlled by the ``action_if_not_found`` and ``default`` keyword arguments. Parameters @@ -45,8 +46,7 @@ def get_project_version(project_name, *, action_if_not_found="warn", default=Non """ if action_if_not_found not in {"ignore", "warn", "raise"}: raise ValueError( - "`action=` keyword must be 'ignore', 'warn', or 'raise'; " - f"got: {action_if_not_found!r}." + f"`action=` keyword must be 'ignore', 'warn', or 'raise'; got: {action_if_not_found!r}." ) try: project_version = version(project_name) @@ -114,7 +114,7 @@ def matches(self, type): return False if not self.is_abstract and self.module not in sys.modules: - # If this isn't an abstract type there can't be sublasses unless + # If this isn't an abstract type there can't be subclasses unless # the module was already imported. return False @@ -140,12 +140,13 @@ class TypeIdentifier: (In principle we could also walk the ``__mro__`` of the type we check and see if we find the superclass by name matching.) """ + def __init__(self, identifiers): self.identifiers = tuple(identifiers) # Fill in type information for later use, sort by identifier (without ~ or @) - self._type_infos = tuple(sorted( - (_TypeInfo(ident) for ident in identifiers), key=lambda ti: ti.identifier - )) + self._type_infos = tuple( + sorted((_TypeInfo(ident) for ident in identifiers), key=lambda ti: ti.identifier) + ) self.is_abstract = any(info.is_abstract for info in self._type_infos) self._idents = frozenset(ti.identifier for ti in self._type_infos) @@ -171,7 +172,7 @@ def encompasses(self, other, subclasscheck=False): # We have the same identifier, check if other represents # subclasses of this. any_subclass = False - for self_ti, other_ti in zip(self._type_infos, other._type_infos): + for self_ti, other_ti in zip(self._type_infos, other._type_infos, strict=True): if self_ti.allow_subclasses == other_ti.allow_subclasses: continue if self_ti.allow_subclasses and not other_ti.allow_subclasses: @@ -192,8 +193,10 @@ def __contains__(self, type): return any(ti.matches(type) for ti in self._type_infos) def __or__(self, other): - """Union of two sets of type identifiers. - """ + """Union of two sets of type identifiers.""" if not isinstance(other, TypeIdentifier): return NotImplemented return TypeIdentifier(set(self.identifiers + other.identifiers)) + + +EMPTY_TYPE_IDENTIFIER = TypeIdentifier([]) diff --git a/tests/test_context.py b/tests/test_context.py index e425e0a..b440fa1 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,5 +1,3 @@ -import pytest - from spatch.backend_system import BackendSystem from spatch.testing import BackendDummy @@ -17,7 +15,7 @@ def test_context_basic(): None, environ_prefix="SPATCH_TEST", default_primary_types=("builtin:int",), - backends=[FloatWithContext()] + backends=[FloatWithContext()], ) # Add a dummy dispatchable function that dispatches on all arguments. @@ -25,20 +23,20 @@ def test_context_basic(): def dummy_func(*args, **kwargs): return "fallback", args, kwargs - _, (ctx, *args), kwargs = dummy_func(1, 1.) + _, (ctx, *args), kwargs = dummy_func(1, 1.0) assert ctx.name == "FloatWithContext" assert set(ctx.types) == {int, float} - assert ctx.dispatch_args == (1, 1.) + assert ctx.dispatch_args == (1, 1.0) assert not ctx.prioritized class float_subclass(float): pass with bs.backend_opts(prioritize=("FloatWithContext",)): - _, (ctx, *args), kwargs = dummy_func(float_subclass(1.)) + _, (ctx, *args), kwargs = dummy_func(float_subclass(1.0)) assert ctx.name == "FloatWithContext" assert set(ctx.types) == {float_subclass} - assert ctx.dispatch_args == (float_subclass(1.),) + assert ctx.dispatch_args == (float_subclass(1.0),) assert ctx.prioritized with bs.backend_opts(type=float): @@ -48,4 +46,3 @@ class float_subclass(float): assert set(ctx.types) == {float} assert ctx.dispatch_args == () assert not ctx.prioritized # not prioritized "just" type enforced - diff --git a/tests/test_priority.py b/tests/test_priority.py index 9ef36da..d31a060 100644 --- a/tests/test_priority.py +++ b/tests/test_priority.py @@ -17,12 +17,14 @@ class IntB2(BackendDummy): secondary_types = () requires_opt_in = False + class FloatB(BackendDummy): name = "FloatB" primary_types = ("builtins:float",) secondary_types = ("builtins:int",) requires_opt_in = False + class FloatBH(BackendDummy): name = "FloatBH" primary_types = ("builtins:float", "builtins:int") @@ -35,6 +37,7 @@ class FloatBH(BackendDummy): higher_priority_than = ("FloatB", "FloatBL") requires_opt_in = False + class FloatBL(BackendDummy): name = "FloatBL" primary_types = ("builtins:float",) @@ -58,24 +61,35 @@ class RealB(BackendDummy): requires_opt_in = False -@pytest.mark.parametrize("backends, expected", [ - ([RealB(), IntB(), IntB2(), FloatB(), IntSubB()], - ["default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB"]), - # Reverse, gives the same order, except for IntB and IntB2 - ([RealB(), IntB(), IntB2(), FloatB(), IntSubB()][::-1], - ["default", "IntB2", "IntB", "IntSubB", "FloatB", "RealB"]), - # And check that manual priority works: - ([RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()], - ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"]), - ([RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()][::-1], - ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"]), -]) +@pytest.mark.parametrize( + ("backends", "expected"), + [ + ( + [RealB(), IntB(), IntB2(), FloatB(), IntSubB()], + ["default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB"], + ), + # Reverse, gives the same order, except for IntB and IntB2 + ( + [RealB(), IntB(), IntB2(), FloatB(), IntSubB()][::-1], + ["default", "IntB2", "IntB", "IntSubB", "FloatB", "RealB"], + ), + # And check that manual priority works: + ( + [RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()], + ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"], + ), + ( + [RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()][::-1], + ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"], + ), + ], +) def test_order_basic(backends, expected): bs = BackendSystem( None, environ_prefix="SPATCH_TEST", default_primary_types=("builtin:int",), - backends=backends + backends=backends, ) order = bs.backend_opts().backends @@ -88,10 +102,9 @@ def bs(): None, environ_prefix="SPATCH_TEST", default_primary_types=("builtin:int",), - backends=[RealB(), IntB(), IntB2(), FloatB(), IntSubB()] + backends=[RealB(), IntB(), IntB2(), FloatB(), IntSubB()], ) - assert bs.backend_opts().backends == ( - "default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB") + assert bs.backend_opts().backends == ("default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB") # Add a dummy dispatchable function that dispatches on all arguments. @bs.dispatchable(None, module="", qualname="dummy_func") @@ -113,14 +126,12 @@ def test_global_opts_basic(bs): def test_opts_context_basic(bs): with bs.backend_opts(prioritize=("RealB",), disable=("IntB", "default")): - assert bs.backend_opts().backends == ( - "RealB", "IntB2", "IntSubB", "FloatB") + assert bs.backend_opts().backends == ("RealB", "IntB2", "IntSubB", "FloatB") assert bs.dummy_func(a=1) == ("RealB", (), {"a": 1}) # Also check nested context, re-enables IntB with bs.backend_opts(prioritize=("IntB",)): - assert bs.backend_opts().backends == ( - "IntB", "RealB", "IntB2", "IntSubB", "FloatB") + assert bs.backend_opts().backends == ("IntB", "RealB", "IntB2", "IntSubB", "FloatB") assert bs.dummy_func(1) == ("IntB", (1,), {})