From 31fd3d2c492b2759110f47557c6c992694e859aa Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 11:26:52 -0500 Subject: [PATCH 01/33] Add initial version of .pre-commit-config.yaml --- .pre-commit-config.yaml | 46 ++++++++++++++++++++++++++++ docs/Makefile | 1 - docs/source/api/for_backends.rst | 2 +- docs/source/api/for_libraries.rst | 1 - src/spatch/_spatch_example/README.md | 2 +- src/spatch/backend_system.py | 4 +-- src/spatch/testing.py | 1 - tests/test_context.py | 1 - 8 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..62f1e7e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,46 @@ +# https://pre-commit.com/ +# +# Before first use: +# +# $ pre-commit install +# +fail_fast: false +default_language_version: + python: python3 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + # Sanity checks + - id: check-added-large-files + - id: check-case-conflict + # - id: check-executables-have-shebangs # No executable files yet + - id: check-illegal-windows-names + - id: check-merge-conflict + # Checks based on file type + - id: check-ast + # - id: check-json # No json files yet + - id: check-symlinks + - id: check-toml + # - id: check-xml # No xml files yet + - id: check-yaml + # Detect mistakes + - id: check-vcs-permalinks + - id: debug-statements + - id: destroyed-symlinks + - id: detect-private-key + - id: forbid-submodules + # Automatic fixes + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + # - id: requirements-txt-fixer # No requirements.txt file yet + - id: trailing-whitespace + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: no-commit-to-branch # No commit directly to main + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/docs/Makefile b/docs/Makefile index 3c400d4..39250bf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -21,4 +21,3 @@ html: Makefile # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - diff --git a/docs/source/api/for_backends.rst b/docs/source/api/for_backends.rst index 281d1bf..128a8a8 100644 --- a/docs/source/api/for_backends.rst +++ b/docs/source/api/for_backends.rst @@ -94,7 +94,7 @@ The following fields are supported for each function: - ``function``: The implementation to dispatch to. - ``should_run`` (optional): A function that gets all inputs (and context) - and can decide to defer. Unless you know things will error, try to make sure + and can decide to defer. Unless you know things will error, try to make sure that this function is light-weight. - ``uses_context``: Whether the implementation needs a ``DispatchContext``. - ``additional_docs`` (optional): Brief text to add to the documentation diff --git a/docs/source/api/for_libraries.rst b/docs/source/api/for_libraries.rst index 3e305dd..c74ffc7 100644 --- a/docs/source/api/for_libraries.rst +++ b/docs/source/api/for_libraries.rst @@ -16,4 +16,3 @@ API to create dispatchable functions .. autoclass:: spatch.backend_system.BackendSystem :class-doc-from: init :members: dispatchable, backend_opts - diff --git a/src/spatch/_spatch_example/README.md b/src/spatch/_spatch_example/README.md index 0401624..779b0d1 100644 --- a/src/spatch/_spatch_example/README.md +++ b/src/spatch/_spatch_example/README.md @@ -113,4 +113,4 @@ hello from backend 2 How types work precisely should be decided by the backend, but care should be taken. E.g. it is not clear if returning a float is OK when the user said `type=complex`. (In the future, we may want to think more about this, especially if `type=complex|real` -may make sense, or if we should fall back if no implementation can be found.) \ No newline at end of file +may make sense, or if we should fall back if no implementation can be found.) diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 33e0b89..79be427 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -698,7 +698,7 @@ def wrap_callable(func): def backend_opts(self): """Property returning a :py:class:`BackendOpts` class specific to this library (tied to this backend system). - """ + """ return type( f"BackendOpts", (BackendOpts,), @@ -969,4 +969,4 @@ def __call__(self, *args, **kwargs): if call_trace is not None: call_trace.append(("default fallback", "called")) - return self._default_func(*args, **kwargs) \ No newline at end of file + return self._default_func(*args, **kwargs) diff --git a/src/spatch/testing.py b/src/spatch/testing.py index de75a75..d8a9c20 100644 --- a/src/spatch/testing.py +++ b/src/spatch/testing.py @@ -36,4 +36,3 @@ def get_function(cls, name, default=None): def dummy_func(cls, *args, **kwargs): # Always define a small function that mainly forwards. return cls.name, args, kwargs - diff --git a/tests/test_context.py b/tests/test_context.py index e425e0a..612bab4 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -48,4 +48,3 @@ class float_subclass(float): assert set(ctx.types) == {float} assert ctx.dispatch_args == () assert not ctx.prioritized # not prioritized "just" type enforced - From 4eb2bc1760d327373a9086f5526028511e56b1cb Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 11:35:07 -0500 Subject: [PATCH 02/33] validate-pyproject --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62f1e7e..065a6c9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,11 @@ repos: args: [--fix=lf] # - id: requirements-txt-fixer # No requirements.txt file yet - id: trailing-whitespace + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.24.1 + hooks: + - id: validate-pyproject + name: Validate pyproject.toml - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: From 92311fed7e5a823fb19c520e7eb6b7c208e22aaf Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 11:37:39 -0500 Subject: [PATCH 03/33] autoflake --- .pre-commit-config.yaml | 6 ++++++ docs/source/conf.py | 1 - src/spatch/backend_system.py | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 065a6c9..ec3e0b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,6 +41,12 @@ repos: hooks: - id: validate-pyproject name: Validate pyproject.toml + # Remove unnecessary imports (currently behaves better than ruff) + - repo: https://github.com/PyCQA/autoflake + rev: v2.3.1 + hooks: + - id: autoflake + args: [--in-place] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/docs/source/conf.py b/docs/source/conf.py index 1fac2e1..476d644 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,7 +1,6 @@ from __future__ import annotations import importlib.metadata -from typing import Any project = "spatch" copyright = "2025, Spatch authors" diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 79be427..20dd628 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -1,4 +1,3 @@ -import contextlib import contextvars import dataclasses from dataclasses import dataclass From c3d3b3fd89f0389e25d6332b53b81ba94b47374b Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 11:40:04 -0500 Subject: [PATCH 04/33] pyupgrade --- .pre-commit-config.yaml | 6 ++++++ src/spatch/_spatch_example/library.py | 1 - src/spatch/backend_system.py | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec3e0b7..5efd50f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,12 @@ repos: hooks: - id: autoflake args: [--in-place] + # Let's keep `pyupgrade` even though `ruff --fix` probably does most of it + - repo: https://github.com/asottile/pyupgrade + rev: v3.20.0 + hooks: + - id: pyupgrade + args: [--py310-plus] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/src/spatch/_spatch_example/library.py b/src/spatch/_spatch_example/library.py index 5b05fec..ec12571 100644 --- a/src/spatch/_spatch_example/library.py +++ b/src/spatch/_spatch_example/library.py @@ -1,4 +1,3 @@ - from spatch.backend_system import BackendSystem _backend_system = BackendSystem( diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 20dd628..debd4bb 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -8,7 +8,8 @@ import sys import textwrap from types import MethodType -from typing import Any, Callable +from typing import Any +from collections.abc import Callable from spatch import from_identifier, get_identifier from spatch.utils import TypeIdentifier, valid_backend_name @@ -919,7 +920,7 @@ def _get_dispatch_args(self, *args, **kwargs): def __call__(self, *args, **kwargs): dispatch_args = tuple(self._get_dispatch_args(*args, **kwargs)) # At this point dispatch_types is not filtered for known types. - dispatch_types = set(type(val) for val in dispatch_args) + dispatch_types = {type(val) for val in dispatch_args} state = self._backend_system._dispatch_state.get() ordered_backends, type_, prioritized, trace = state From ae57dc87dbe9bcda76250bd9bfffeda8db1478a5 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 11:44:36 -0500 Subject: [PATCH 05/33] black (line-length = 100) --- .pre-commit-config.yaml | 5 ++ pyproject.toml | 4 + src/spatch/_spatch_example/backend.py | 5 +- src/spatch/_spatch_example/library.py | 3 +- src/spatch/backend_system.py | 113 +++++++++++++++----------- src/spatch/backend_utils.py | 5 +- src/spatch/testing.py | 4 +- src/spatch/utils.py | 10 +-- tests/test_context.py | 10 +-- tests/test_priority.py | 51 +++++++----- 10 files changed, 126 insertions(+), 84 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5efd50f..d4a39ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,6 +53,11 @@ repos: hooks: - id: pyupgrade args: [--py310-plus] + # black often looks better than ruff-format + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index fe93614..cc39c85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,5 +59,9 @@ addopts = [ "--doctest-glob=docs/source/**.md", ] +[tool.black] +line-length = 100 +target-version = ["py310", "py311", "py312", "py313"] + [tool.coverage] run.source = ["spatch"] diff --git a/src/spatch/_spatch_example/backend.py b/src/spatch/_spatch_example/backend.py index cb0f307..55d14fc 100644 --- a/src/spatch/_spatch_example/backend.py +++ b/src/spatch/_spatch_example/backend.py @@ -1,6 +1,7 @@ try: from spatch.backend_utils import BackendImplementation except ModuleNotFoundError: # pragma: no cover + class Noop: # No-operation/do nothing version of a BackendImplementation def __call__(self, *args, **kwargs): @@ -15,9 +16,9 @@ def __call__(self, *args, **kwargs): backend1 = BackendImplementation("backend1") backend2 = BackendImplementation("backend2") + # For backend 1 -@backend1.implements( - library.divide, uses_context=True, should_run=lambda info, x, y: True) +@backend1.implements(library.divide, uses_context=True, should_run=lambda info, x, y: True) def divide(context, x, y): """This implementation works well on floats.""" print("hello from backend 1") diff --git a/src/spatch/_spatch_example/library.py b/src/spatch/_spatch_example/library.py index ec12571..66e7027 100644 --- a/src/spatch/_spatch_example/library.py +++ b/src/spatch/_spatch_example/library.py @@ -3,12 +3,13 @@ _backend_system = BackendSystem( "_spatch_example_backends", # entry point group "_SPATCH_EXAMPLE_BACKENDS", # environment variable prefix - default_primary_types=["builtins:int"] + default_primary_types=["builtins:int"], ) backend_opts = _backend_system.backend_opts + @_backend_system.dispatchable(["x", "y"]) def divide(x, y): """Divide integers, other types may be supported via backends""" diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index debd4bb..4a67c9a 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -23,7 +23,7 @@ class Backend: name: str primary_types: TypeIdentifier = TypeIdentifier([]) secondary_types: TypeIdentifier = TypeIdentifier([]) - functions : dict = dataclasses.field(default_factory=dict) + functions: dict = dataclasses.field(default_factory=dict) known_backends: frozenset = frozenset() higher_priority_than: frozenset = frozenset() lower_priority_than: frozenset = frozenset() @@ -106,7 +106,8 @@ def compare_backends(backend1, backend2, prioritize_over): raise RuntimeError( "Backends {backend1.name} and {backend2.name} report inconsistent " "priorities (this means they are buggy). You can manually set " - "a priority or remove one of the backends.") + "a priority or remove one of the backends." + ) if cmp1 > cmp2: return cmp1 else: @@ -140,14 +141,14 @@ def _modified_state( elif unknown_backends == "ignore": pass else: - raise ValueError( - "_modified_state() unknown_backends must be raise or ignore") + raise ValueError("_modified_state() unknown_backends must be raise or ignore") if type is not None: if not backend_system.known_type(type, primary=True): raise ValueError( f"Type '{type}' not a valid primary type of any backend. " - "It is impossible to enforce use of this type for any function.") + "It is impossible to enforce use of this type for any function." + ) ordered_backends, _, prioritized, curr_trace = curr_state prioritized = prioritized | frozenset(prioritize) @@ -291,7 +292,11 @@ def __init__(self, *, prioritize=(), disable=(), type=None, trace=False): self._state = _modified_state( self._backend_system, self._dispatch_state.get(), - prioritize=prioritize, disable=disable, type=type, trace=trace) + prioritize=prioritize, + disable=disable, + type=type, + trace=trace, + ) # unpack new state to provide information: self.backends, self.prioritized, self.type, self.trace = self._state self._token = None @@ -300,13 +305,14 @@ def __repr__(self): # We could allow a repr that can be copy pasted, but this seems more clear? inactive = tuple(b for b in self._backend_system.backends if b not in self.backends) type_str = " type: {tuple(self.type)[0]!r}\n" if self.type else "" - return (f"" - ) + return ( + f"" + ) def enable_globally(self): """Apply these backend options globally. @@ -323,12 +329,13 @@ def enable_globally(self): # If the state was never set or the state matches (ignoring trace) # and there was no trace registered before this is OK. Otherwise warn. if curr_state is not None and ( - curr_state[:-1] != self._state[:-1] - or curr_state[-1] is not None): + curr_state[:-1] != self._state[:-1] or curr_state[-1] is not None + ): warnings.warn( "Backend options were previously modified, global change of the " "backends state should only be done once from the main program.", - UserWarning, 2 + UserWarning, + 2, ) self._token = self._dispatch_state.set(self._state) @@ -374,6 +381,7 @@ def func(): func : callable The decorated function. """ + # In this form, allow entering multiple times by storing the token # inside the wrapper functions locals @functools.wraps(func) @@ -427,8 +435,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N self.backends = {} if default_primary_types is not None: self.backends["default"] = Backend( - name="default", - primary_types=TypeIdentifier(default_primary_types) + name="default", primary_types=TypeIdentifier(default_primary_types) ) try: @@ -442,12 +449,14 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N raise ValueError( f"Invalid order with duplicate backend in environment " f"variable {environ_prefix}_SET_ORDER:\n" - f" {orders_str}") + f" {orders_str}" + ) prev_b = None for b in orders: if not valid_backend_name(b): raise ValueError( - f"Name {b!r} in {environ_prefix}_SET_ORDER is not a valid backend name.") + f"Name {b!r} in {environ_prefix}_SET_ORDER is not a valid backend name." + ) if prev_b is not None: prioritize_over.setdefault(prev_b, set()).add(b) # If an opposite prioritization was already set, discard it. @@ -459,7 +468,8 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N warnings.warn( f"Ignoring invalid environment variable {environ_prefix}_SET_ORDER " f"due to error: {e}", - UserWarning, 2 + UserWarning, + 2, ) try: @@ -468,12 +478,14 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N for b in prioritize: if not valid_backend_name(b): raise ValueError( - f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name.") + f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name." + ) except Exception as e: warnings.warn( f"Ignoring invalid environment variable {environ_prefix}_PRIORITIZE " f"due to error: {e}", - UserWarning, 2 + UserWarning, + 2, ) try: @@ -482,12 +494,14 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N for b in blocked: if not b.isidentifier(): raise ValueError( - f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name.") + f"Name {b!r} in {environ_prefix}_PRIORITIZE is not a valid backend name." + ) except Exception as e: warnings.warn( f"Ignoring invalid environment variable {environ_prefix}_SET_ORDER " f"due to error: {e}", - UserWarning, 2 + UserWarning, + 2, ) # Note that the order of adding backends matters, we add `backends` first @@ -500,10 +514,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N try: self.backend_from_namespace(backend) except Exception as e: - warnings.warn( - f"Skipping backend {backend.name} due to error: {e}", - UserWarning, 2 - ) + warnings.warn(f"Skipping backend {backend.name} due to error: {e}", UserWarning, 2) # Create a directed graph for which backends have a known higher priority than others. # The topological sorter is stable with respect to the original order, so we add @@ -515,7 +526,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N backends = [self.backends[n] for n in graph] for i, b1 in enumerate(backends): - for b2 in backends[i+1:]: + for b2 in backends[i + 1 :]: cmp = compare_backends(b1, b2, prioritize_over) if cmp < 0: graph[b1.name].append(b2.name) @@ -531,9 +542,9 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N base_state = (order, None, frozenset(), None) disable = {b.name for b in self.backends.values() if b.requires_opt_in} state = _modified_state( - self, base_state, prioritize=prioritize, disable=disable, unknown_backends="ignore") - self._dispatch_state = contextvars.ContextVar( - f"{group}.dispatch_state", default=state) + self, base_state, prioritize=prioritize, disable=disable, unknown_backends="ignore" + ) + self._dispatch_state = contextvars.ContextVar(f"{group}.dispatch_state", default=state) @staticmethod def _toposort(graph): @@ -549,9 +560,10 @@ def visit(node, order, _visiting={}): f"Backends form a priority cycle. This is a bug in a backend or your\n" f"environment settings. Check the environment variable {environ_prefix}_SET_ORDER\n" f"and change it for example to:\n" - f" {environ_prefix}_SET_ORDER=\"{cycle[-1]}>{cycle[-2]}\"\n" + f' {environ_prefix}_SET_ORDER="{cycle[-1]}>{cycle[-2]}"\n' f"to break the offending cycle:\n" - f" {'>'.join(cycle)}") from None + f" {'>'.join(cycle)}" + ) from None _visiting[node] = None # mark as visiting/in-progress for n in graph[node]: @@ -584,13 +596,11 @@ def _get_entry_points(group, blocked): namespace = ep.load() if ep.name != namespace.name: raise RuntimeError( - f"Entrypoint name {ep.name!r} and actual name {namespace.name!r} mismatch.") + f"Entrypoint name {ep.name!r} and actual name {namespace.name!r} mismatch." + ) backends.append(namespace) except Exception as e: - warnings.warn( - f"Skipping backend {ep.name} due to error: {e}", - UserWarning, 3 - ) + warnings.warn(f"Skipping backend {ep.name} due to error: {e}", UserWarning, 3) return sorted(backends, key=lambda x: x.name) @@ -636,7 +646,8 @@ def backend_from_namespace(self, info_namespace): if new_backend.name in self.backends: warnings.warn( f"Backend of name '{new_backend.name}' already exists. Ignoring second!", - UserWarning, 3 + UserWarning, + 3, ) return self.backends[new_backend.name] = new_backend @@ -681,6 +692,7 @@ def dispatchable(self, dispatch_args=None, *, module=None, qualname=None): Unfortunately, changing the module can confuse some tools, so we may wish to change the behavior of actually overriding it. """ + def wrap_callable(func): # Overwrite original module (we use it later, could also pass it) if module is not None: @@ -749,6 +761,7 @@ class DispatchContext: to use this to decide that e.g. a NumPy array will be converted to a cupy array, but only if prioritized. """ + # The idea is for the context to be very light-weight so that specific # information should be properties (because most likely we will never need it). # This object can grow to provide more information to backends. @@ -843,8 +856,8 @@ def __init__(self, backend_system, func, dispatch_args, ident=None): if p.name not in dispatch_args: continue if ( - p.kind == inspect.Parameter.POSITIONAL_ONLY or - p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + p.kind == inspect.Parameter.POSITIONAL_ONLY + or p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD ): # Accepting it as a keyword is irrelevant here (fails later) new_dispatch_args[p.name] = i @@ -852,7 +865,8 @@ def __init__(self, backend_system, func, dispatch_args, ident=None): new_dispatch_args[p.name] = sys.maxsize else: raise TypeError( - f"Parameter {p.name} is variable. Must use callable `dispatch_args`.") + f"Parameter {p.name} is variable. Must use callable `dispatch_args`." + ) if len(dispatch_args) != len(new_dispatch_args): not_found = set(dispatch_args) - set(new_dispatch_args) @@ -887,7 +901,10 @@ def __init__(self, backend_system, func, dispatch_args, ident=None): # Create implementations, lazy loads should_run (and maybe more in the future). self._implementations = _Implentations(impl_infos) self._implementations["default"] = _Implementation( - "default", self._default_func, None, False, + "default", + self._default_func, + None, + False, ) if not new_doc: @@ -913,7 +930,8 @@ def _get_dispatch_args(self, *args, **kwargs): return args + tuple(kwargs.values()) else: return tuple( - val for name, pos in self._dispatch_args.items() + val + for name, pos in self._dispatch_args.items() if (val := args[pos] if pos < len(args) else kwargs.get(name)) is not None ) @@ -929,7 +947,8 @@ def __call__(self, *args, **kwargs): dispatch_types = frozenset(dispatch_types) dispatch_types, matching_backends = self._backend_system.get_types_and_backends( - dispatch_types, ordered_backends) + dispatch_types, ordered_backends + ) if trace is not None: call_trace = [] diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index 635566d..5e92747 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -20,8 +20,7 @@ class BackendImplementation: impl_to_info: dict[str, BackendFunctionInfo] def __init__(self, backend_name: str): - """Helper class to create backends. - """ + """Helper class to create backends.""" self.name = backend_name self.api_to_info = {} # {api_identity_string: backend_function_info} self.impl_to_info = {} # {impl_identity_string: backend_function_info} @@ -163,7 +162,7 @@ def __repr__(self): return "\n".join( [ "(", - *(repr(line + '\n') for line in self.lines[:-1]), + *(repr(line + "\n") for line in self.lines[:-1]), repr(self.lines[-1]), ")", ] diff --git a/src/spatch/testing.py b/src/spatch/testing.py index d8a9c20..ef42a78 100644 --- a/src/spatch/testing.py +++ b/src/spatch/testing.py @@ -1,5 +1,6 @@ from spatch.utils import get_identifier + class _FuncGetter: def __init__(self, get): self.get = get @@ -11,6 +12,7 @@ class BackendDummy: Forwards any lookup to the class. Documentation are used from the function which must match in the name. """ + def __init__(self): self.functions = _FuncGetter(self.get_function) @@ -20,7 +22,7 @@ def get_function(cls, name, default=None): _, name = name.split(":") # Not get_identifier because it would find the super-class name. - res = {"function": f"{cls.__module__}:{cls.__name__}.{name}" } + res = {"function": f"{cls.__module__}:{cls.__name__}.{name}"} if hasattr(cls, "uses_context"): res["uses_context"] = cls.uses_context if hasattr(cls, "should_run"): diff --git a/src/spatch/utils.py b/src/spatch/utils.py index c82a25d..44c0036 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -140,12 +140,13 @@ class TypeIdentifier: (In principle we could also walk the ``__mro__`` of the type we check and see if we find the superclass by name matching.) """ + def __init__(self, identifiers): self.identifiers = tuple(identifiers) # Fill in type information for later use, sort by identifier (without ~ or @) - self._type_infos = tuple(sorted( - (_TypeInfo(ident) for ident in identifiers), key=lambda ti: ti.identifier - )) + self._type_infos = tuple( + sorted((_TypeInfo(ident) for ident in identifiers), key=lambda ti: ti.identifier) + ) self.is_abstract = any(info.is_abstract for info in self._type_infos) self._idents = frozenset(ti.identifier for ti in self._type_infos) @@ -192,8 +193,7 @@ def __contains__(self, type): return any(ti.matches(type) for ti in self._type_infos) def __or__(self, other): - """Union of two sets of type identifiers. - """ + """Union of two sets of type identifiers.""" if not isinstance(other, TypeIdentifier): return NotImplemented return TypeIdentifier(set(self.identifiers + other.identifiers)) diff --git a/tests/test_context.py b/tests/test_context.py index 612bab4..79b0c36 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -17,7 +17,7 @@ def test_context_basic(): None, environ_prefix="SPATCH_TEST", default_primary_types=("builtin:int",), - backends=[FloatWithContext()] + backends=[FloatWithContext()], ) # Add a dummy dispatchable function that dispatches on all arguments. @@ -25,20 +25,20 @@ def test_context_basic(): def dummy_func(*args, **kwargs): return "fallback", args, kwargs - _, (ctx, *args), kwargs = dummy_func(1, 1.) + _, (ctx, *args), kwargs = dummy_func(1, 1.0) assert ctx.name == "FloatWithContext" assert set(ctx.types) == {int, float} - assert ctx.dispatch_args == (1, 1.) + assert ctx.dispatch_args == (1, 1.0) assert not ctx.prioritized class float_subclass(float): pass with bs.backend_opts(prioritize=("FloatWithContext",)): - _, (ctx, *args), kwargs = dummy_func(float_subclass(1.)) + _, (ctx, *args), kwargs = dummy_func(float_subclass(1.0)) assert ctx.name == "FloatWithContext" assert set(ctx.types) == {float_subclass} - assert ctx.dispatch_args == (float_subclass(1.),) + assert ctx.dispatch_args == (float_subclass(1.0),) assert ctx.prioritized with bs.backend_opts(type=float): diff --git a/tests/test_priority.py b/tests/test_priority.py index 9ef36da..036a35e 100644 --- a/tests/test_priority.py +++ b/tests/test_priority.py @@ -17,12 +17,14 @@ class IntB2(BackendDummy): secondary_types = () requires_opt_in = False + class FloatB(BackendDummy): name = "FloatB" primary_types = ("builtins:float",) secondary_types = ("builtins:int",) requires_opt_in = False + class FloatBH(BackendDummy): name = "FloatBH" primary_types = ("builtins:float", "builtins:int") @@ -35,6 +37,7 @@ class FloatBH(BackendDummy): higher_priority_than = ("FloatB", "FloatBL") requires_opt_in = False + class FloatBL(BackendDummy): name = "FloatBL" primary_types = ("builtins:float",) @@ -58,24 +61,35 @@ class RealB(BackendDummy): requires_opt_in = False -@pytest.mark.parametrize("backends, expected", [ - ([RealB(), IntB(), IntB2(), FloatB(), IntSubB()], - ["default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB"]), - # Reverse, gives the same order, except for IntB and IntB2 - ([RealB(), IntB(), IntB2(), FloatB(), IntSubB()][::-1], - ["default", "IntB2", "IntB", "IntSubB", "FloatB", "RealB"]), - # And check that manual priority works: - ([RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()], - ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"]), - ([RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()][::-1], - ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"]), -]) +@pytest.mark.parametrize( + "backends, expected", + [ + ( + [RealB(), IntB(), IntB2(), FloatB(), IntSubB()], + ["default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB"], + ), + # Reverse, gives the same order, except for IntB and IntB2 + ( + [RealB(), IntB(), IntB2(), FloatB(), IntSubB()][::-1], + ["default", "IntB2", "IntB", "IntSubB", "FloatB", "RealB"], + ), + # And check that manual priority works: + ( + [RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()], + ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"], + ), + ( + [RealB(), IntB(), FloatB(), FloatBH(), FloatBL(), IntSubB()][::-1], + ["default", "IntB", "IntSubB", "FloatBH", "FloatB", "FloatBL", "RealB"], + ), + ], +) def test_order_basic(backends, expected): bs = BackendSystem( None, environ_prefix="SPATCH_TEST", default_primary_types=("builtin:int",), - backends=backends + backends=backends, ) order = bs.backend_opts().backends @@ -88,10 +102,9 @@ def bs(): None, environ_prefix="SPATCH_TEST", default_primary_types=("builtin:int",), - backends=[RealB(), IntB(), IntB2(), FloatB(), IntSubB()] + backends=[RealB(), IntB(), IntB2(), FloatB(), IntSubB()], ) - assert bs.backend_opts().backends == ( - "default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB") + assert bs.backend_opts().backends == ("default", "IntB", "IntB2", "IntSubB", "FloatB", "RealB") # Add a dummy dispatchable function that dispatches on all arguments. @bs.dispatchable(None, module="", qualname="dummy_func") @@ -113,14 +126,12 @@ def test_global_opts_basic(bs): def test_opts_context_basic(bs): with bs.backend_opts(prioritize=("RealB",), disable=("IntB", "default")): - assert bs.backend_opts().backends == ( - "RealB", "IntB2", "IntSubB", "FloatB") + assert bs.backend_opts().backends == ("RealB", "IntB2", "IntSubB", "FloatB") assert bs.dummy_func(a=1) == ("RealB", (), {"a": 1}) # Also check nested context, re-enables IntB with bs.backend_opts(prioritize=("IntB",)): - assert bs.backend_opts().backends == ( - "IntB", "RealB", "IntB2", "IntSubB", "FloatB") + assert bs.backend_opts().backends == ("IntB", "RealB", "IntB2", "IntSubB", "FloatB") assert bs.dummy_func(1) == ("IntB", (1,), {}) From 22d2a42e8374684d716397e1d719a15fbbe788cc Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 12:08:47 -0500 Subject: [PATCH 06/33] ruff --- .pre-commit-config.yaml | 5 +++ pyproject.toml | 79 ++++++++++++++++++++++++++++++++++++ src/spatch/backend_system.py | 4 +- tests/test_context.py | 2 - 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d4a39ba..6566563 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,6 +58,11 @@ repos: rev: 25.1.0 hooks: - id: black + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.4 + hooks: + - id: ruff + args: [--fix-only, --show-fixes] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index cc39c85..f0a68ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,5 +63,84 @@ addopts = [ line-length = 100 target-version = ["py310", "py311", "py312", "py313"] +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +unfixable = [ + "F841", # unused-variable (Note: can leave useless expression) + "B905", # zip-without-explicit-strict (Note: prefer `zip(x, y, strict=True)`) +] +ignore = [ + # Maybe consider + "ANN", # flake8-annotations (We don't fully use annotations yet) + "B904", # Use `raise from` to specify exception cause (Note: sometimes okay to raise original exception) + "PERF401", # Use a list comprehension to create a transformed list (Note: poorly implemented atm) + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` (Note: no annotations yet) + "RUF021", # parenthesize-chained-operators (Note: results don't look good yet) + "RUF023", # unsorted-dunder-slots (Note: maybe fine, but noisy changes) + "S310", # Audit URL open for permitted schemes (Note: we don't download URLs in normal usage) + "TRY004", # Prefer `TypeError` exception for invalid type (Note: good advice, but not worth the nuisance) + + # Intentionally ignored + "COM812", # Trailing comma missing + "D203", # 1 blank line required before class docstring (Note: conflicts with D211, which is preferred) + "D213", # (Note: conflicts with D212, which is preferred) + "D400", # First line should end with a period (Note: prefer D415, which also allows "?" and "!") + "N801", # Class name ... should use CapWords convention (Note:we have a few exceptions to this) + "N802", # Function name ... should be lowercase + "N803", # Argument name ... should be lowercase (Maybe okay--except in tests) + "N806", # Variable ... in function should be lowercase + "N807", # Function name should not start and end with `__` + "N818", # Exception name ... should be named with an Error suffix (Note: good advice) + "PERF203", # `try`-`except` within a loop incurs performance overhead (Note: too strict) + "PLC0205", # Class `__slots__` should be a non-string iterable (Note: string is fine) + "PLR0124", # Name compared with itself, consider replacing `x == x` (Note: too strict) + "PLR0911", # Too many return statements + "PLR0912", # Too many branches + "PLR0913", # Too many arguments to function call + "PLR0915", # Too many statements + "PLR2004", # Magic number used in comparison, consider replacing magic with a constant variable + "PLW0603", # Using the global statement to update ... is discouraged (Note: yeah, discouraged, but too strict) + "PLW0642", # Reassigned `self` variable in instance method (Note: too strict for us) + "PLW2901", # Outer for loop variable ... overwritten by inner assignment target (Note: good advice, but too strict) + "RET502", # Do not implicitly `return None` in function able to return non-`None` value + "RET503", # Missing explicit `return` at the end of function able to return non-`None` value + "RET504", # Unnecessary variable assignment before `return` statement + "RUF018", # Avoid assignment expressions in `assert` statements + "S110", # `try`-`except`-`pass` detected, consider logging the exception (Note: good advice, but we don't log) + "S112", # `try`-`except`-`continue` detected, consider logging the exception (Note: good advice, but we don't log) + "S603", # `subprocess` call: check for execution of untrusted input (Note: not important for us) + "S607", # Starting a process with a partial executable path (Note: not important for us) + "SIM102", # Use a single `if` statement instead of nested `if` statements (Note: often necessary) + "SIM105", # Use contextlib.suppress(...) instead of try-except-pass (Note: try-except-pass is much faster) + "SIM108", # Use ternary operator ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) + "TRY003", # Avoid specifying long messages outside the exception class (Note: why?) + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` (Note: using `|` is slower atm) + + # Ignored categories + "C90", # mccabe (Too strict, but maybe we should make things less complex) + "BLE", # flake8-blind-except (Maybe consider) + "FBT", # flake8-boolean-trap (Why?) + "DJ", # flake8-django (We don't use django) + "EM", # flake8-errmsg (Perhaps nicer, but too much work) + "PYI", # flake8-pyi (We don't have stub files yet) + "SLF", # flake8-self (We can use our own private variables--sheesh!) + "TCH", # flake8-type-checking (Note: figure out type checking later) + "ARG", # flake8-unused-arguments (Sometimes helpful, but too strict) + "TD", # flake8-todos (Maybe okay to add some of these) + "FIX", # flake8-fixme (like flake8-todos) + "ERA", # eradicate (We like code in comments!) + "PD", # pandas-vet (Intended for scripts that use pandas, not libraries) +] + +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false + +[tool.ruff.lint.pydocstyle] +convention = "numpy" + [tool.coverage] run.source = ["spatch"] diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 4a67c9a..b953c8f 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -712,7 +712,7 @@ def backend_opts(self): (tied to this backend system). """ return type( - f"BackendOpts", + "BackendOpts", (BackendOpts,), {"_dispatch_state": self._dispatch_state, "_backend_system": self}, ) @@ -981,7 +981,7 @@ def __call__(self, *args, **kwargs): # useful if `can_run` was passed only cachable parameters (e.g. `method="meth"`, # or even `backend=`, although that would be special). # (We may tag on a reason for a non-True return value as well or use context.) - raise NotImplementedError(f"Currently, should run must return True or False.") + raise NotImplementedError("Currently, should run must return True or False.") elif trace is not None and impl.should_run is not None: call_trace.append((name, "skipped due to should_run returning False")) diff --git a/tests/test_context.py b/tests/test_context.py index 79b0c36..b440fa1 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,5 +1,3 @@ -import pytest - from spatch.backend_system import BackendSystem from spatch.testing import BackendDummy From 2394cd16531dbbf29a44e01cd057a517a9faad75 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 13:00:48 -0500 Subject: [PATCH 07/33] ruff extend-select (no changes) --- pyproject.toml | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f0a68ec..b2bcdc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,51 @@ line-length = 100 target-version = "py310" [tool.ruff.lint] +extend-select = [ + # Defaults from https://github.com/scientific-python/cookie + "B", # flake8-bugbear + # "I", # isort + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + # "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "TID251", # flake8-tidy-imports.banned-api + "T20", # flake8-print + "UP", # pyupgrade + "YTT", # flake8-2020 + + # Additional ones (may be unnecessary, low value, or a nuisance). + # It's okay to experiment and add or remove checks from here + "S", # bandit + "A", # flake8-builtins + "COM", # flake8-commas + "DTZ", # flake8-datetimez + "T10", # flake8-debugger + "EXE", # flake8-executable + "ISC", # flake8-implicit-str-concat + "G", # flake8-logging-format + "INP", # flake8-no-pep420 + "Q", # flake8-quotes + "RSE", # flake8-raise + "NPY", # NumPy-specific rules + "N", # pep8-naming + "PLC", # pylint Convention + "PLE", # pylint Error + "PLR", # pylint Refactor + "PLW", # pylint Warning + "E", # pycodestyle Error + "W", # pycodestyle Warning + "F", # pyflakes + "TRY", # tryceratops +] unfixable = [ "F841", # unused-variable (Note: can leave useless expression) "B905", # zip-without-explicit-strict (Note: prefer `zip(x, y, strict=True)`) @@ -124,11 +169,9 @@ ignore = [ "BLE", # flake8-blind-except (Maybe consider) "FBT", # flake8-boolean-trap (Why?) "DJ", # flake8-django (We don't use django) - "EM", # flake8-errmsg (Perhaps nicer, but too much work) "PYI", # flake8-pyi (We don't have stub files yet) "SLF", # flake8-self (We can use our own private variables--sheesh!) "TCH", # flake8-type-checking (Note: figure out type checking later) - "ARG", # flake8-unused-arguments (Sometimes helpful, but too strict) "TD", # flake8-todos (Maybe okay to add some of these) "FIX", # flake8-fixme (like flake8-todos) "ERA", # eradicate (We like code in comments!) From 0dcc8cf8ed3498b0c748f6b173700b5cd9dc5c27 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 13:04:59 -0500 Subject: [PATCH 08/33] isort via ruff --- pyproject.toml | 2 +- src/spatch/_spatch_example/entry_point.py | 2 +- src/spatch/_spatch_example/entry_point2.py | 2 +- src/spatch/backend_system.py | 10 +++++----- src/spatch/backend_utils.py | 2 +- src/spatch/utils.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b2bcdc1..d1f81ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ target-version = "py310" extend-select = [ # Defaults from https://github.com/scientific-python/cookie "B", # flake8-bugbear - # "I", # isort + "I", # isort "ARG", # flake8-unused-arguments "C4", # flake8-comprehensions "EM", # flake8-errmsg diff --git a/src/spatch/_spatch_example/entry_point.py b/src/spatch/_spatch_example/entry_point.py index b0906bd..791e021 100644 --- a/src/spatch/_spatch_example/entry_point.py +++ b/src/spatch/_spatch_example/entry_point.py @@ -24,7 +24,7 @@ if __name__ == "__main__": # pragma: no cover # Run this file as a script to update this file - from spatch.backend_utils import update_entrypoint from spatch._spatch_example.backend import backend1 + from spatch.backend_utils import update_entrypoint update_entrypoint(__file__, backend1, "spatch._spatch_example.backend") diff --git a/src/spatch/_spatch_example/entry_point2.py b/src/spatch/_spatch_example/entry_point2.py index 5831b7f..9355d64 100644 --- a/src/spatch/_spatch_example/entry_point2.py +++ b/src/spatch/_spatch_example/entry_point2.py @@ -19,7 +19,7 @@ if __name__ == "__main__": # pragma: no cover # Run this file as a script to update this file - from spatch.backend_utils import update_entrypoint from spatch._spatch_example.backend import backend2 + from spatch.backend_utils import update_entrypoint update_entrypoint(__file__, backend2, "spatch._spatch_example.backend") diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index b953c8f..f740c24 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -1,20 +1,20 @@ import contextvars import dataclasses -from dataclasses import dataclass import functools import os -import importlib_metadata -import warnings import sys import textwrap +import warnings +from collections.abc import Callable +from dataclasses import dataclass from types import MethodType from typing import Any -from collections.abc import Callable + +import importlib_metadata from spatch import from_identifier, get_identifier from spatch.utils import TypeIdentifier, valid_backend_name - __doctest_skip__ = ["BackendOpts.__init__"] diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index 5e92747..b82cf86 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass from collections.abc import Callable +from dataclasses import dataclass from .utils import from_identifier, get_identifier diff --git a/src/spatch/utils.py b/src/spatch/utils.py index 44c0036..5806867 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -1,9 +1,9 @@ -from importlib import import_module -from importlib.metadata import version -from dataclasses import dataclass, field import re import sys import warnings +from dataclasses import dataclass, field +from importlib import import_module +from importlib.metadata import version def get_identifier(obj): From d7b2a4f46cefac695e8524956caf4f536bc96776 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 13:06:19 -0500 Subject: [PATCH 09/33] flake8-return via ruff --- pyproject.toml | 2 +- src/spatch/backend_system.py | 43 ++++++++++++++++-------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1f81ca..ba2d122 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ extend-select = [ "PL", # pylint "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib - # "RET", # flake8-return + "RET", # flake8-return "RUF", # Ruff-specific "SIM", # flake8-simplify "TID251", # flake8-tidy-imports.banned-api diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index f740c24..8456d54 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -54,10 +54,9 @@ def from_namespace(cls, info): def known_type(self, dispatch_type): if dispatch_type in self.primary_types: return "primary" # TODO: maybe make it an enum? - elif dispatch_type in self.secondary_types: + if dispatch_type in self.secondary_types: return "secondary" - else: - return False + return False def matches(self, dispatch_types): matches = frozenset(self.known_type(t) for t in dispatch_types) @@ -69,7 +68,7 @@ def compare_with_other(self, other): # NOTE: This function is a symmetric comparison if other.name in self.higher_priority_than: return 2 - elif other.name in self.lower_priority_than: + if other.name in self.lower_priority_than: return -2 # If our primary types are a subset of the other, we match more @@ -87,7 +86,7 @@ def compare_backends(backend1, backend2, prioritize_over): # Environment variable prioritization beats everything: if (prio := prioritize_over.get(backend1.name)) and backend2.name in prio: return 3 - elif (prio := prioritize_over.get(backend2.name)) and backend1.name in prio: + if (prio := prioritize_over.get(backend2.name)) and backend1.name in prio: return -3 # Sort by the backends compare function (i.e. type hierarchy and manual order). @@ -97,9 +96,9 @@ def compare_backends(backend1, backend2, prioritize_over): cmp2 = backend2.compare_with_other(backend1) if cmp1 is NotImplemented and cmp2 is NotImplemented: return 0 - elif cmp1 is NotImplemented: + if cmp1 is NotImplemented: return -cmp2 - elif cmp2 is NotImplemented: + if cmp2 is NotImplemented: return cmp1 if cmp1 == cmp2: @@ -110,8 +109,7 @@ def compare_backends(backend1, backend2, prioritize_over): ) if cmp1 > cmp2: return cmp1 - else: - return -cmp2 + return -cmp2 def _modified_state( @@ -138,7 +136,7 @@ def _modified_state( if b not in backend_system.backends: if unknown_backends == "raise": raise ValueError(f"Backend '{b}' not found.") - elif unknown_backends == "ignore": + if unknown_backends == "ignore": pass else: raise ValueError("_modified_state() unknown_backends must be raise or ignore") @@ -792,10 +790,9 @@ def function(self): _function = self._function if type(_function) is not str: return _function - else: - _function = from_identifier(_function) - self._function = _function - return _function + _function = from_identifier(_function) + self._function = _function + return _function class _Implentations(dict): @@ -928,12 +925,11 @@ def _get_dispatch_args(self, *args, **kwargs): # Return all dispatch args if self._dispatch_args is None: return args + tuple(kwargs.values()) - else: - return tuple( - val - for name, pos in self._dispatch_args.items() - if (val := args[pos] if pos < len(args) else kwargs.get(name)) is not None - ) + return tuple( + val + for name, pos in self._dispatch_args.items() + if (val := args[pos] if pos < len(args) else kwargs.get(name)) is not None + ) def __call__(self, *args, **kwargs): dispatch_args = tuple(self._get_dispatch_args(*args, **kwargs)) @@ -972,17 +968,16 @@ def __call__(self, *args, **kwargs): if impl.uses_context: return impl.function(context, *args, **kwargs) - else: - return impl.function(*args, **kwargs) + return impl.function(*args, **kwargs) - elif should_run is not False: + if should_run is not False: # Strict to allow future use as "should run if needed only". That would merge # "can" and "should" run. I can see a dedicated `can_run`, but see it as more # useful if `can_run` was passed only cachable parameters (e.g. `method="meth"`, # or even `backend=`, although that would be special). # (We may tag on a reason for a non-True return value as well or use context.) raise NotImplementedError("Currently, should run must return True or False.") - elif trace is not None and impl.should_run is not None: + if trace is not None and impl.should_run is not None: call_trace.append((name, "skipped due to should_run returning False")) if call_trace is not None: From d885624f015ef4b302cae3d862d93667fc34bd61 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:12:15 -0500 Subject: [PATCH 10/33] Enable ruff linting, and some fixes --- .pre-commit-config.yaml | 4 ++++ pyproject.toml | 26 ++++++++++++++++++++---- src/spatch/backend_system.py | 39 +++++++++++++++--------------------- src/spatch/backend_utils.py | 7 ++++--- src/spatch/utils.py | 5 ++++- tests/test_priority.py | 2 +- 6 files changed, 51 insertions(+), 32 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6566563..244ad6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,6 +63,10 @@ repos: hooks: - id: ruff args: [--fix-only, --show-fixes] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.4 + hooks: + - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index ba2d122..28171b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,12 +69,10 @@ target-version = "py310" [tool.ruff.lint] extend-select = [ - # Defaults from https://github.com/scientific-python/cookie + # Defaults from https://github.com/scientific-python/cookie excluding ARG and EM "B", # flake8-bugbear "I", # isort - "ARG", # flake8-unused-arguments "C4", # flake8-comprehensions - "EM", # flake8-errmsg "ICN", # flake8-import-conventions "PGH", # pygrep-hooks "PIE", # flake8-pie @@ -112,6 +110,10 @@ extend-select = [ "W", # pycodestyle Warning "F", # pyflakes "TRY", # tryceratops + + # Maybe consider + # "EM", # flake8-errmsg (Perhaps nicer, but it's a lot of work) + # "ALL", # to see everything! ] unfixable = [ "F841", # unused-variable (Note: can leave useless expression) @@ -125,10 +127,12 @@ ignore = [ "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` (Note: no annotations yet) "RUF021", # parenthesize-chained-operators (Note: results don't look good yet) "RUF023", # unsorted-dunder-slots (Note: maybe fine, but noisy changes) - "S310", # Audit URL open for permitted schemes (Note: we don't download URLs in normal usage) + "S310", # Audit URL open for permitted schemes (Note: we don't download URLs in normal usage) "TRY004", # Prefer `TypeError` exception for invalid type (Note: good advice, but not worth the nuisance) + "TRY301", # Abstract `raise` to an inner function # Intentionally ignored + "B019", # Use of `functools.lru_cache` or `functools.cache` on methods can lead to memory leaks "COM812", # Trailing comma missing "D203", # 1 blank line required before class docstring (Note: conflicts with D211, which is preferred) "D213", # (Note: conflicts with D212, which is preferred) @@ -141,6 +145,7 @@ ignore = [ "N818", # Exception name ... should be named with an Error suffix (Note: good advice) "PERF203", # `try`-`except` within a loop incurs performance overhead (Note: too strict) "PLC0205", # Class `__slots__` should be a non-string iterable (Note: string is fine) + "PLC0415", # `import` should be at the top-level of a file (Note: good advice, too strict) "PLR0124", # Name compared with itself, consider replacing `x == x` (Note: too strict) "PLR0911", # Too many return statements "PLR0912", # Too many branches @@ -172,12 +177,25 @@ ignore = [ "PYI", # flake8-pyi (We don't have stub files yet) "SLF", # flake8-self (We can use our own private variables--sheesh!) "TCH", # flake8-type-checking (Note: figure out type checking later) + "ARG", # flake8-unused-arguments (Sometimes helpful, but too strict) "TD", # flake8-todos (Maybe okay to add some of these) "FIX", # flake8-fixme (like flake8-todos) "ERA", # eradicate (We like code in comments!) "PD", # pandas-vet (Intended for scripts that use pandas, not libraries) ] +[tool.ruff.lint.per-file-ignores] +"src/spatch/**/__init__.py" = ["F401"] # Allow unused import (w/o defining `__all__`) +"src/spatch/_spatch_example/backend.py" = ["T201"] # Allow print +"docs/**/*.py" = ["INP001"] # Not a package +"tests/*.py" = [ + "S101", # Allow assert + "INP001", # Not a package +] + +[tool.ruff.lint.flake8-builtins] +builtins-ignorelist = ["copyright", "type"] + [tool.ruff.lint.flake8-pytest-style] fixture-parentheses = false mark-parentheses = false diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 8456d54..fcaf8b2 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -13,7 +13,7 @@ import importlib_metadata from spatch import from_identifier, get_identifier -from spatch.utils import TypeIdentifier, valid_backend_name +from spatch.utils import EMPTY_TYPE_IDENTIFIER, TypeIdentifier, valid_backend_name __doctest_skip__ = ["BackendOpts.__init__"] @@ -21,8 +21,8 @@ @dataclass(slots=True) class Backend: name: str - primary_types: TypeIdentifier = TypeIdentifier([]) - secondary_types: TypeIdentifier = TypeIdentifier([]) + primary_types: TypeIdentifier = EMPTY_TYPE_IDENTIFIER + secondary_types: TypeIdentifier = EMPTY_TYPE_IDENTIFIER functions: dict = dataclasses.field(default_factory=dict) known_backends: frozenset = frozenset() higher_priority_than: frozenset = frozenset() @@ -60,9 +60,7 @@ def known_type(self, dispatch_type): def matches(self, dispatch_types): matches = frozenset(self.known_type(t) for t in dispatch_types) - if "primary" in matches and False not in matches: - return True - return False + return "primary" in matches and False not in matches def compare_with_other(self, other): # NOTE: This function is a symmetric comparison @@ -501,6 +499,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N UserWarning, 2, ) + self._environ_prefix = environ_prefix # Note that the order of adding backends matters, we add `backends` first # and then entry point ones in alphabetical order. @@ -544,21 +543,21 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N ) self._dispatch_state = contextvars.ContextVar(f"{group}.dispatch_state", default=state) - @staticmethod - def _toposort(graph): + def _toposort(self, graph): # Adapted from Wikipedia's depth-first pseudocode. We are not using graphlib, # because it doesn't preserve the original order correctly. # This depth-first approach does preserve it. - def visit(node, order, _visiting={}): + def visit(node, order, _visiting): if node in order: return if node in _visiting: - cycle = (tuple(_visiting.keys()) + (node,))[::-1] + cycle = (*_visiting.keys(), node)[::-1] raise RuntimeError( f"Backends form a priority cycle. This is a bug in a backend or your\n" - f"environment settings. Check the environment variable {environ_prefix}_SET_ORDER\n" + "environment settings. Check the environment variable " + f"{self._environ_prefix}_SET_ORDER\n" f"and change it for example to:\n" - f' {environ_prefix}_SET_ORDER="{cycle[-1]}>{cycle[-2]}"\n' + f' {self._environ_prefix}_SET_ORDER="{cycle[-1]}>{cycle[-2]}"\n' f"to break the offending cycle:\n" f" {'>'.join(cycle)}" ) from None @@ -570,10 +569,10 @@ def visit(node, order, _visiting={}): del _visiting[node] order[node] = None # add sorted node - to_sort = list(graph.keys()) + # to_sort = list(graph.keys()) # @eriknw to @seberg: should this be used? order = {} # dict as a sorted set for n in list(graph.keys()): - visit(n, order) + visit(n, order, {}) return tuple(order.keys()) @@ -604,10 +603,7 @@ def _get_entry_points(group, blocked): @functools.lru_cache(maxsize=128) def known_type(self, dispatch_type, primary=False): - for backend in self.backends.values(): - if backend.known_type(dispatch_type): - return True - return False + return any(backend.known_type(dispatch_type) for backend in self.backends.values()) def get_known_unique_types(self, dispatch_types): # From a list of args, return only the set of dispatch types @@ -852,13 +848,10 @@ def __init__(self, backend_system, func, dispatch_args, ident=None): for i, p in enumerate(sig.parameters.values()): if p.name not in dispatch_args: continue - if ( - p.kind == inspect.Parameter.POSITIONAL_ONLY - or p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD - ): + if p.kind in {p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD}: # Accepting it as a keyword is irrelevant here (fails later) new_dispatch_args[p.name] = i - elif p.kind == inspect.Parameter.KEYWORD_ONLY: + elif p.kind == p.KEYWORD_ONLY: new_dispatch_args[p.name] = sys.maxsize else: raise TypeError( diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index b82cf86..70a1035 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -1,5 +1,6 @@ from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path from .utils import from_identifier, get_identifier @@ -119,7 +120,7 @@ def set_should_run(self, backend_func: str | Callable): def inner(func: Callable): identity = get_identifier(func) - if identity.endswith(":") or identity.endswith(":_"): + if identity.endswith((":", ":_")): backend_func._should_run = func identity = f"{impl_identity}._should_run" info = self.impl_to_info[impl_identity] @@ -234,8 +235,8 @@ def update_entrypoint( ] # Step 4: replace text - with open(filepath) as f: + with Path(filepath).open("r") as f: text = f.read() text = update_text(text, lines, "functions", indent=indent) - with open(filepath, "w") as f: + with Path(filepath).open("w") as f: f.write(text) diff --git a/src/spatch/utils.py b/src/spatch/utils.py index 5806867..bcc06b2 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -172,7 +172,7 @@ def encompasses(self, other, subclasscheck=False): # We have the same identifier, check if other represents # subclasses of this. any_subclass = False - for self_ti, other_ti in zip(self._type_infos, other._type_infos): + for self_ti, other_ti in zip(self._type_infos, other._type_infos, strict=True): if self_ti.allow_subclasses == other_ti.allow_subclasses: continue if self_ti.allow_subclasses and not other_ti.allow_subclasses: @@ -197,3 +197,6 @@ def __or__(self, other): if not isinstance(other, TypeIdentifier): return NotImplemented return TypeIdentifier(set(self.identifiers + other.identifiers)) + + +EMPTY_TYPE_IDENTIFIER = TypeIdentifier([]) diff --git a/tests/test_priority.py b/tests/test_priority.py index 036a35e..d31a060 100644 --- a/tests/test_priority.py +++ b/tests/test_priority.py @@ -62,7 +62,7 @@ class RealB(BackendDummy): @pytest.mark.parametrize( - "backends, expected", + ("backends", "expected"), [ ( [RealB(), IntB(), IntB2(), FloatB(), IntSubB()], From 3f6acab99d762654ac35fb0f47e60013959b745c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:26:14 -0500 Subject: [PATCH 11/33] pyroma --- .pre-commit-config.yaml | 9 +++++++-- pyproject.toml | 25 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 244ad6c..8082dee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,12 +61,17 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.4 hooks: - - id: ruff + - id: ruff-check args: [--fix-only, --show-fixes] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.4 hooks: - - id: ruff + - id: ruff-check + - repo: https://github.com/regebro/pyroma + rev: "5.0" + hooks: + - id: pyroma + args: [-n, "9", .] # Need author email to score a 10 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 28171b6..b34e2ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,34 @@ authors = [ {name = "Scientific Python Developers"}, ] dynamic = ["version"] -description = "Coming soon" +description = "Coming soon: Python library for enabling dispatching to backends" readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.10" dependencies = ["importlib_metadata"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = ["dispatching"] + +[project.urls] +homepage = "https://github.com/scientific-python/spatch" +documentation = "https://scientific-python.github.io/spatch" +source = "https://github.com/scientific-python/spatch" +changelog = "https://github.com/scientific-python/spatch/releases" [dependency-groups] # black is used to format entry-point files From 5441abe40b528fc678b69e46ce41782d91003f38 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:30:52 -0500 Subject: [PATCH 12/33] SIM102 (manually) --- src/spatch/backend_system.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index fcaf8b2..30fb266 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -139,12 +139,11 @@ def _modified_state( else: raise ValueError("_modified_state() unknown_backends must be raise or ignore") - if type is not None: - if not backend_system.known_type(type, primary=True): - raise ValueError( - f"Type '{type}' not a valid primary type of any backend. " - "It is impossible to enforce use of this type for any function." - ) + if type is not None and not backend_system.known_type(type, primary=True): + raise ValueError( + f"Type '{type}' not a valid primary type of any backend. " + "It is impossible to enforce use of this type for any function." + ) ordered_backends, _, prioritized, curr_trace = curr_state prioritized = prioritized | frozenset(prioritize) From 7dc1d04f25ccbaa43ababe0ce9fc8a93061881de Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:40:34 -0500 Subject: [PATCH 13/33] codespell --- .pre-commit-config.yaml | 8 ++++++++ docs/source/api/for_users.rst | 8 ++++---- docs/source/design_choices.md | 8 ++++---- src/spatch/_spatch_example/README.md | 2 +- src/spatch/backend_system.py | 12 ++++++------ src/spatch/utils.py | 2 +- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8082dee..5a14b5c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,10 +63,18 @@ repos: hooks: - id: ruff-check args: [--fix-only, --show-fixes] + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + types_or: [python, markdown, rst, toml, yaml] + additional_dependencies: [tomli] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.4 hooks: - id: ruff-check + # `pyroma` may help keep our package standards up to date if best practices change. + # This is probably a "low value" check though and safe to remove if we want faster pre-commit. - repo: https://github.com/regebro/pyroma rev: "5.0" hooks: diff --git a/docs/source/api/for_users.rst b/docs/source/api/for_users.rst index 4623849..9e48ddf 100644 --- a/docs/source/api/for_users.rst +++ b/docs/source/api/for_users.rst @@ -18,13 +18,13 @@ Libraries will re-expose all of this functionality under their own API/names. There are currently three global environment variables to modify dispatching behavior at startup time: -* ``_PRIORITIZE``: Comma seperated list of backends. +* ``_PRIORITIZE``: Comma separated list of backends. This is the same as :py:class:`BackendOpts` ``prioritize=`` option. -* ``_BLOCK``: Comma seperated list of backends. +* ``_BLOCK``: Comma separated list of backends. This prevents loading the backends as if they were not installed. No backend code will be executed (its entry-point will not be loaded). -* ``_SET_ORDER``: Comma seperated list of backend orders. - seperated by ``>``. I.e. ``name1>name2,name3>name2`` means that ``name1`` +* ``_SET_ORDER``: Comma separated list of backend orders. + separated by ``>``. I.e. ``name1>name2,name3>name2`` means that ``name1`` and ``name3`` are ordered before ``name2``. This is more fine-grained than the above two and the above two take precedence. Useful to fix relative order of backends without affecting the priority of backends not listed diff --git a/docs/source/design_choices.md b/docs/source/design_choices.md index 302ed5b..f4c7dc0 100644 --- a/docs/source/design_choices.md +++ b/docs/source/design_choices.md @@ -76,7 +76,7 @@ square it. 2. Use the GPU for NumPy arrays (i.e. make code faster hopefully without changing behavior significantly). 3. Enforce _unsafe_ use of `cupy` arrays ("zero code change" story) - because point 2. would incure unnecessary and slow host/device copies. + because point 2. would incur unnecessary and slow host/device copies. The first use-case is implicit: The user never has to do anything, it will always just work. @@ -228,7 +228,7 @@ We would agree that piggy backing on an existing dunder approach (such as the Array API) seems nice. But ultimately it would be far less flexible (Array API has no backend selection) and be more complex since the Array API would have to provide infrastructure that -can deal with aribtrary libraries different backends for each of them. +can deal with arbitrary libraries different backends for each of them. To us, it seems simply the wrong way around: The library should dispatch and it can dispatch to a function that uses the Array API. @@ -277,7 +277,7 @@ use-case seems just far more relevant. We think the reason for this are: `import cupy as np` then it is to also modify many library imports. ```{admonition} Future note -`spatch` endevors to provide a more explicit path (and if this gets outdated, maybe we do). +`spatch` endeavors to provide a more explicit path (and if this gets outdated, maybe we do). We expect this to be more of the form of `library.function.invoke(state, ...)` or also `state.invoke(library.function, ...)` or `library.function.invoke(backend=)(...)`. @@ -330,7 +330,7 @@ neither backend authors or end-users need to understand the "how". Of course, one can avoid this complexity by just asking backends to fix the order where if it matters. -We believe that the current complexity in spatch is managable, although we would agree that +We believe that the current complexity in spatch is manageable, although we would agree that a library that isn't dedicated to dispatching should likely avoid it. diff --git a/src/spatch/_spatch_example/README.md b/src/spatch/_spatch_example/README.md index 779b0d1..a7a5bdc 100644 --- a/src/spatch/_spatch_example/README.md +++ b/src/spatch/_spatch_example/README.md @@ -95,7 +95,7 @@ hello from backend 1 0.5 >>> with backend_opts(type=complex): ... # backen 2 returning a float for complex "input" is probably OK -... # (but may be debateable) +... # (but may be debatable) ... divide(1, 2) hello from backend 2 0.5 diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 30fb266..2598889 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -89,7 +89,7 @@ def compare_backends(backend1, backend2, prioritize_over): # Sort by the backends compare function (i.e. type hierarchy and manual order). # We default to a type based comparisons but allow overriding this, so check - # both ways (to find the overriding). This also find inconcistencies. + # both ways (to find the overriding). This also find inconsistencies. cmp1 = backend1.compare_with_other(backend2) cmp2 = backend2.compare_with_other(backend1) if cmp1 is NotImplemented and cmp2 is NotImplemented: @@ -229,7 +229,7 @@ def __init__(self, *, prioritize=(), disable=(), type=None, trace=False): type The type to dispatch for within this context. trace - The trace object (currenly a list as described in the examples). + The trace object (currently a list as described in the examples). If used, the trace is also returned when entering the context. Notes @@ -275,7 +275,7 @@ def __init__(self, *, prioritize=(), disable=(), type=None, trace=False): Backends should simply document their behavior with ``backend_opts`` and which usage pattern they see for their users. - Tracing calls can be done using, where ``trace`` is a list of informations for + Tracing calls can be done using, where ``trace`` is a list of information for each call. This contains a tuple of the function identifier and a list of backends called (typically exactly one, but it will also note if a backend deferred via ``should_run``). @@ -318,7 +318,7 @@ def enable_globally(self): is safer to use the contextmanager ``with`` statement instead. This method will issue a warning if the - dispatching state has been previously modified programatically. + dispatching state has been previously modified programmatically. """ curr_state = self._dispatch_state.get(None) # None used before default # If the state was never set or the state matches (ignoring trace) @@ -533,7 +533,7 @@ def __init__(self, group, environ_prefix, default_primary_types=None, backends=N # Finalize backends to be a dict sorted by priority. self.backends = {b: self.backends[b] for b in order} - # The state is the ordered (active) backends and the prefered type (None) + # The state is the ordered (active) backends and the preferred type (None) # and the trace (None as not tracing). base_state = (order, None, frozenset(), None) disable = {b.name for b in self.backends.values() if b.requires_opt_in} @@ -965,7 +965,7 @@ def __call__(self, *args, **kwargs): if should_run is not False: # Strict to allow future use as "should run if needed only". That would merge # "can" and "should" run. I can see a dedicated `can_run`, but see it as more - # useful if `can_run` was passed only cachable parameters (e.g. `method="meth"`, + # useful if `can_run` was passed only cacheable parameters (e.g. `method="meth"`, # or even `backend=`, although that would be special). # (We may tag on a reason for a non-True return value as well or use context.) raise NotImplementedError("Currently, should run must return True or False.") diff --git a/src/spatch/utils.py b/src/spatch/utils.py index bcc06b2..8996080 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -114,7 +114,7 @@ def matches(self, type): return False if not self.is_abstract and self.module not in sys.modules: - # If this isn't an abstract type there can't be sublasses unless + # If this isn't an abstract type there can't be subclasses unless # the module was already imported. return False From b3846dc36e96939fea63268beb69f5675f6b35be Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:46:24 -0500 Subject: [PATCH 14/33] taplo to auto-format pyproject.toml --- .pre-commit-config.yaml | 14 +++++++++----- pyproject.toml | 25 ++++++++++--------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a14b5c..667d55e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,15 +75,19 @@ repos: - id: ruff-check # `pyroma` may help keep our package standards up to date if best practices change. # This is probably a "low value" check though and safe to remove if we want faster pre-commit. - - repo: https://github.com/regebro/pyroma - rev: "5.0" + # - repo: https://github.com/regebro/pyroma + # rev: "5.0" + # hooks: + # - id: pyroma + # args: [-n, "9", .] # Need author email to score a 10 + - repo: https://github.com/ComPWA/taplo-pre-commit + rev: v0.9.3 hooks: - - id: pyroma - args: [-n, "9", .] # Need author email to score a 10 + - id: taplo-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - - id: no-commit-to-branch # No commit directly to main + - id: no-commit-to-branch # No commit directly to main - repo: meta hooks: - id: check-hooks-apply diff --git a/pyproject.toml b/pyproject.toml index b34e2ea..94aed9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,7 @@ build-backend = "hatchling.build" [project] name = "spatch" -authors = [ - {name = "Scientific Python Developers"}, -] +authors = [{ name = "Scientific Python Developers" }] dynamic = ["version"] description = "Coming soon: Python library for enabling dispatching to backends" readme = "README.md" @@ -46,10 +44,7 @@ test = [ "pytest-doctestplus", { include-group = "backend_utils" }, ] -dev = [ - { include-group = "test" }, - { include-group = "docs" }, -] +dev = [{ include-group = "test" }, { include-group = "docs" }] docs = [ "sphinx>=7.0", "sphinx-copybutton", @@ -73,14 +68,12 @@ backend2 = 'spatch._spatch_example.entry_point2' [tool.pytest.ini_options] doctest_plus = "enabled" testpaths = [ - "tests", - "src/spatch", # for doc testing - "docs", + "tests", + "src/spatch", # for doc testing + "docs", ] norecursedirs = ["src"] -addopts = [ - "--doctest-glob=docs/source/**.md", -] +addopts = ["--doctest-glob=docs/source/**.md"] [tool.black] line-length = 100 @@ -208,8 +201,10 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -"src/spatch/**/__init__.py" = ["F401"] # Allow unused import (w/o defining `__all__`) -"src/spatch/_spatch_example/backend.py" = ["T201"] # Allow print +"src/spatch/**/__init__.py" = [ + "F401", # Allow unused import (w/o defining `__all__`) +] +"src/spatch/_spatch_example/backend.py" = ["T201"] # Allow print "docs/**/*.py" = ["INP001"] # Not a package "tests/*.py" = [ "S101", # Allow assert From 55529cc5ec4fbbe7b9ecd7bce9f5b3a7c897c0b6 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:48:41 -0500 Subject: [PATCH 15/33] yamllint --- .pre-commit-config.yaml | 4 ++++ .yamllint.yaml | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 .yamllint.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 667d55e..69e1c54 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,6 +84,10 @@ repos: rev: v0.9.3 hooks: - id: taplo-format + - repo: https://github.com/adrienverge/yamllint + rev: v1.37.1 + hooks: + - id: yamllint - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..fb66779 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,5 @@ +--- +extends: default +rules: + document-start: disable + line-length: disable From 00806b520970bdbae930fa53090cd4cbb46566d0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:49:41 -0500 Subject: [PATCH 16/33] prettier --- .github/workflows/build-docs.yml | 6 +- .pre-commit-config.yaml | 4 + README.md | 40 +++++---- docs/source/design_choices.md | 118 +++++++++++++++------------ src/spatch/_spatch_example/README.md | 27 ++++-- 5 files changed, 115 insertions(+), 80 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 70bc210..56af3c1 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: deploy: - description: 'Publish docs?' + description: "Publish docs?" required: true type: boolean @@ -58,8 +58,8 @@ jobs: if: inputs.deploy permissions: - pages: write # to deploy to Pages - id-token: write # to verify the deployment originates from an appropriate source + pages: write # to deploy to Pages + id-token: write # to verify the deployment originates from an appropriate source # Deploy to the github-pages environment environment: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69e1c54..8ca17db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,6 +80,10 @@ repos: # hooks: # - id: pyroma # args: [-n, "9", .] # Need author email to score a 10 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.6.2 + hooks: + - id: prettier - repo: https://github.com/ComPWA/taplo-pre-commit rev: v0.9.3 hooks: diff --git a/README.md b/README.md index ac43ba7..e17adc6 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ based on feedback.** Spatch is a dispatching tool with a focus on scientific python libraries. It integrates two forms of dispatching into a single backend system: -* Type dispatching for the main type used by a library. + +- Type dispatching for the main type used by a library. In the scientific python world, this is often the array object. -* Backend selection to offer alternative implementations to users. +- Backend selection to offer alternative implementations to users. These may be faster or less precise, but using them should typically not change code behavior drastically. @@ -26,7 +27,7 @@ to a large scale deployment. Unfortunately, providing code for a host of such types isn't easy and the original library authors usually have neither the bandwidth nor -expertise to do it. Additionally, such layers would have to be optional +expertise to do it. Additionally, such layers would have to be optional components of the library. `spatch` allows for a solution to this dilemma by allowing a third party @@ -35,7 +36,7 @@ to enable library functions to work with alternative types. It should be noted that spatch is not a generic multiple dispatching library. It is opinionated about being strictly typed (we can and probably will support subclasses in the future, though). -It also considers all arguments identically. I.e. if a function takes +It also considers all arguments identically. I.e. if a function takes two inputs (of the kind we dispatch for), there is no distinction for their order. Besides these two things, `spatch` is however a typical type dispatching @@ -56,11 +57,12 @@ For example, we may have a faster algorithm that is parallelized while the old one was not. Or an implementation that dispatches to the GPU but still returns NumPy arrays (as the library always did). -Backend selection _modifies_ behavior rather than extending it. In some +Backend selection _modifies_ behavior rather than extending it. In some cases those modifications may be small (maybe it is really only faster). For the user, backend _selection_ often means that they should explicitly select a preferred backend (e.g. over the default implementation). This could be for example via a context manager: + ```python with backend_opts(prioritize="gpu_backend"): library.function() # now running on the GPU @@ -76,36 +78,38 @@ with backend_opts(prioritize="gpu_backend"): it should be considered a prototype when it comes to API stability. Some examples for missing things we are still working on: -* No way to conveniently see which backends may be used when calling a + +- No way to conveniently see which backends may be used when calling a function (rather than actually calling it). And probably more inspection utilities. -* We have implemented the ability for a backend to defer and not run, +- We have implemented the ability for a backend to defer and not run, but not the ability to run anyway if there is no alternative. -* The main library implementation currently can't distinguish fallback +- The main library implementation currently can't distinguish fallback and default path easily. It should be easy to do this (two functions, `should_run`, or just via `uses_context`). -* `spatch` is very much designed to be fast but that doesn't mean it - is particularly fast yet. We may need to optimize parts (potentially +- `spatch` is very much designed to be fast but that doesn't mean it + is particularly fast yet. We may need to optimize parts (potentially lowering parts to a compiled language). -* We have not implemented tools to test backends, e.g. against parts - of the original library. We expect that "spatch" actually includes most - tools to do this. For example, we could define a `convert` function +- We have not implemented tools to test backends, e.g. against parts + of the original library. We expect that "spatch" actually includes most + tools to do this. For example, we could define a `convert` function that backends can implement to convert arguments in tests as needed. There are also many smaller or bigger open questions and those include whether the API proposed here is actually quite what we want. Other things are for example whether we want API like: -* `dispatchable.invoke(type=, backend=)`. -* Maybe libraries should use `like=` in functions that take no dispatchable + +- `dispatchable.invoke(type=, backend=)`. +- Maybe libraries should use `like=` in functions that take no dispatchable arguments. -* How do we do classes such as scikit-learn estimators. A simple solution might +- How do we do classes such as scikit-learn estimators. A simple solution might a `get_backend(...)` dispatching explicitly once. But we could use more involved schemes, rather remembering the dispatching state of the `.fit()`. We can also see many small conveniences, for example: -* Extract the dispatchable arguments from type annotations. -* Support a magic `Defer` return, rather than the `should_run` call. +- Extract the dispatchable arguments from type annotations. +- Support a magic `Defer` return, rather than the `should_run` call. # Usage examples diff --git a/docs/source/design_choices.md b/docs/source/design_choices.md index f4c7dc0..a219274 100644 --- a/docs/source/design_choices.md +++ b/docs/source/design_choices.md @@ -1,33 +1,35 @@ -`spatch` design choices -======================= +# `spatch` design choices This document is designed as a companion to reading the normal API documentation to answer why the API is designed as it is (not detail questions such as naming). Please always remember that `spatch` serves two distinct use-cases: -* Type dispatching, which extending functionality to new types. E.g. without the backend + +- Type dispatching, which extending functionality to new types. E.g. without the backend you only support NumPy, but with the backend you also support CuPy. -* Alternative backend selection where the backend provides an alternative - implementation that is for example faster. It may even use the GPU +- Alternative backend selection where the backend provides an alternative + implementation that is for example faster. It may even use the GPU but the user might still only work with NumPy arrays. A backend can choose to serve one or both of these. Particularly important design considerations are: -* Light-weight import. -* Fast dispatching especially when the user has backends installed but the code + +- Light-weight import. +- Fast dispatching especially when the user has backends installed but the code is not using them. -* No magic by default: Just installing a backend should _not_ change behavior. +- No magic by default: Just installing a backend should _not_ change behavior. (This largely requires backend discipline and coordination with libraries.) -* Adopting `spatch` should be easy for libraries. -* We should aim to make backend authors lives easy but not by making that +- Adopting `spatch` should be easy for libraries. +- We should aim to make backend authors lives easy but not by making that of library authors much harder. % This section really just to keep the first TOC a bit cleaner... Specific design questions/reasons ---------------------------------- -The following are some specific questions/arguments. The answers may be +--- + +The following are some specific questions/arguments. The answers may be rambling at times :). ```{contents} @@ -38,13 +40,14 @@ rambling at times :). `spatch` uses entry-points like plugin systems and also like NetworkX. Entry-points allow us to do a few things: -* Good user API: If the backend is installed and adds CuPy support, the user + +- Good user API: If the backend is installed and adds CuPy support, the user has to change no code at all. Since the backend is not included neither in the library nor `cupy` entry-points - are the only solution to this. (For type dispatching.) -* Introspection and documentation: Users can introspect available backends + are the only solution to this. (For type dispatching.) +- Introspection and documentation: Users can introspect available backends right away and the list cannot magically change at run-time due to an import. -* We can push many decisions (such as backend priority order) to the startup time. +- We can push many decisions (such as backend priority order) to the startup time. ### Use of identifier strings for everything @@ -63,7 +66,7 @@ This needs to be done with care, but is useful locally (e.g. array creation with no inputs) or just for experiments. If we think of choosing backends, it may come easier to think about it in terms -of a name. For example `skimage` could have a backend `"cucim"` and that +of a name. For example `skimage` could have a backend `"cucim"` and that backend adds `cupy` support. So the question is why shouldn't the user just activate the cucim backend with @@ -72,6 +75,7 @@ its name and that might change behavior of functions to return cupy arrays? We tried to do this and believe wasted a lot of time trying to find a way to square it. `"cucim"` users might in principle want to do any of three things: + 1. Add `cupy` support to `skimage`, i.e. type dispatching for CuPy. 2. Use the GPU for NumPy arrays (i.e. make code faster hopefully without changing behavior significantly). @@ -81,30 +85,32 @@ square it. The first use-case is implicit: The user never has to do anything, it will always just work. But use-cases 2 and 3, both require some explicit opt-in via -`with backend_opts(...)`. We could have distinguished these two with two backend +`with backend_opts(...)`. We could have distinguished these two with two backend names so users would either do `with backend_opts("cucim[numpy]")` or `with backend_opts("cucim[cupy]")`. And that may be easier to understand for users. But unfortunately, it is trying to reduce a two dimensional problem into a one dimensional one and this lead to a long tail of paper-cuts: -* You still need to educate users about `cucim[numpy]` and `cucim[cupy]` and it + +- You still need to educate users about `cucim[numpy]` and `cucim[cupy]` and it is no easier for the backend to implement both. Does `cucim` need 2-3 backends that look similar? Or a mechanism to have multiple - names for one backend? But then the type logic inside the backend depends on the + names for one backend? But then the type logic inside the backend depends on the name in complicated ways, while in `spatch` it depends exactly on `DispatchContext.types`. -* It isn't perfectly clear what happens if `cucim[cupy]` is missing a function. +- It isn't perfectly clear what happens if `cucim[cupy]` is missing a function. Maybe there is another backend to run, but it won't know about the `[cupy]` information! -* If there was a backend that also takes cupy arrays, but has a faster, less-precise +- If there was a backend that also takes cupy arrays, but has a faster, less-precise version, the user would have to activate both `cucim-fast[cupy]` and `cucim[cupy]` to get the desired behavior. We firmly believe that teaching users about `cucim[cupy]` or some backend specific (unsafe) option is not significantly easier than teaching them to use: -* `with backend_opts(prioritize="cucim"):` for the safe first case and -* `with backend_opts(type=cupy.ndarray):` for the unsafe second case. + +- `with backend_opts(prioritize="cucim"):` for the safe first case and +- `with backend_opts(type=cupy.ndarray):` for the unsafe second case. (May also include a `priority="cucim"` as well.) And this is much more explicit about the desire ("use cupy") while avoiding above @@ -136,7 +142,7 @@ fallback (or backend) that uses a custom `library.convert(...)` function and calls the default implementation. If libraries are very interested in this, we should consider extending -`spatch` here. But otherwise, we think it should be backends authors taking the +`spatch` here. But otherwise, we think it should be backends authors taking the lead, although that might end up in extending spatch. ### No generic catch-all implementations? @@ -148,7 +154,7 @@ For example, NumPy has `__array_ufunc__` and because all ufuncs are similar Dask can have a single implementation that always works! If there are well structured use-cases (like the NumPy ufunc one) a library -_can_ choose to explicitly support it: Rather than dispatching for `np.add`, +_can_ choose to explicitly support it: Rather than dispatching for `np.add`, you would dispatch for `np.ufunc()` and pass `np.add` as an argument. In general, we accept that this may be useful and could be a future addition. @@ -165,6 +171,7 @@ The reason for this design is convenience, speed, and simplicity. Matching on types is the only truly fast thing, because it allows a design where the decision of which backend to call is done by: + 1. Fetching the current dispatching state. This is very fast, but unavoidable as we must check user configuration. 2. Figuring out which (unique) types the user passed to do type dispatching. @@ -183,7 +190,7 @@ Caching, type-safety, and no accidentally slow behavior explains why However, we could still ask each backend to provide a `matches(types)` function rather than asking them to list primary and secondary types. -This choice is just for convenience right now. Since we insist on types we might +This choice is just for convenience right now. Since we insist on types we might as well handle the these things inside `spatch`. The reason for "primary" and "secondary" type is to make backends simple if @@ -196,14 +203,16 @@ And if you just want a single type, then ignore the "secondary" one... ### How would I create for example an "Array API" backend? -This is actually not a problem at all. But the backend will have to provide +This is actually not a problem at all. But the backend will have to provide an abstract base class and make sure it is importable in a very light-weight way: + ```python class SupportsArrayAPI(abc.ABC): @classmethod def __subclasshook__(cls, C): return hasattr(C, "__array_nanespace__") ``` + Then you can use `"@mymodule:SupportsArrayAPI"` _almost_ like the normal use. ### Why not use dunder methods (like NumPy, NetworkX, or Array API)? @@ -212,13 +221,14 @@ Magic dunders (`__double_underscore_method__`) are a great way to implement type dispatching (it's also how Python operators work)! But it cannot serve our use-case (and also see advantages of entry-points). The reasons for this is that: -* For NumPy/NetworkX, CuPy is the one that provides the type and thus can attach a + +- For NumPy/NetworkX, CuPy is the one that provides the type and thus can attach a magic dunder to it and provide the implementations for all functions. But for `spatch` the implementation would be a third party (and not e.g. CuPy). And a third party can't easily attach a dunder (e.g. to `cupy.ndarray`) It would not be reliable or work well with the entry-point path to avoid costly - imports. Rather than `spatch` providing the entry-point, cupy would have to. -* Dunders really don't solve the backend selection problem. If they wanted to, + imports. Rather than `spatch` providing the entry-point, cupy would have to. +- Dunders really don't solve the backend selection problem. If they wanted to, you would need another layer pushing the backend selection into types. This may be possible, but would require the whole infrastructure to be centered around the type (i.e. `cupy.ndarray`) rather than the library @@ -229,7 +239,7 @@ Array API) seems nice. But ultimately it would be far less flexible (Array API has no backend selection) and be more complex since the Array API would have to provide infrastructure that can deal with arbitrary libraries different backends for each of them. -To us, it seems simply the wrong way around: The library should dispatch and it +To us, it seems simply the wrong way around: The library should dispatch and it can dispatch to a function that uses the Array API. ### Context manager to modify the dispatching options/state @@ -247,33 +257,38 @@ Context managers simply serve this use-case nicely, quickly, and locally. #### Why not a namespace for explicit dispatching? Based on ideas from NEP 37, the Array API for example dispatches once and then the user has an -`xp` namespace they can pass around. That is great! +`xp` namespace they can pass around. That is great! + +But it is not targeted to end-users. An end-users should write: -But it is not targeted to end-users. An end-users should write: ``` library.function(cupy_array) ``` + and not: + ``` libx = library.dispatch(cupy_array) libx.function(cupy_array) ``` + The second is very explicit and allows to explicitly pass around a "dispatched" state -(i.e. the `libx` namespace). This is can be amazing to write a function that +(i.e. the `libx` namespace). This is can be amazing to write a function that wants to work with different inputs because by using `libx` you don't have to worry about anything and you can even pass it around. -So we love this concept! But we think that the first "end-user" style use-case has to +So we love this concept! But we think that the first "end-user" style use-case has to be center stage for `spatch`. For the core libraries (i.e. NumPy vs. CuPy) the explicit library -use-case seems just far more relevant. We think the reason for this are: -* It is just simpler there, since there no risk of having multiple such contexts. -* Backend selection is not a thing, if it was NumPy or CuPy should naturally handle it +use-case seems just far more relevant. We think the reason for this are: + +- It is just simpler there, since there no risk of having multiple such contexts. +- Backend selection is not a thing, if it was NumPy or CuPy should naturally handle it themselves. (Tis refers to backend selection for speed, not type dispatching. NumPy _does_ type dispatch to cupy and while unsafe, it would not be unfathomable to ask NumPy to dispatch even creation function within a context.) -* User need: It seems much more practical for end-users to just use cupy maybe via +- User need: It seems much more practical for end-users to just use cupy maybe via `import cupy as np` then it is to also modify many library imports. ```{admonition} Future note @@ -288,7 +303,7 @@ We do not consider a "dispatched namespace" to be worth the complexity at this p ### The backend priority seems complicated/not complex enough? We need to decide on a priority for backends, which is not an exact science. -For backend-selection there is no hard-and-fast rule: Normally an alternative +For backend-selection there is no hard-and-fast rule: Normally an alternative implementation should have lower priority unless all sides agree it is drop-in enough to be always used. @@ -300,7 +315,7 @@ B accepts, then we must prefer the B because it matches more precisely This is the equivalence to the Python binary operator rule "subclass before superclass". This rule is important, for example because B may be trying to correct behavior of A. -Now accepts "superclass" in the above is a rather broad term. For example backend A +Now accepts "superclass" in the above is a rather broad term. For example backend A may accept `numpy.ndarray | cupy.ndarray` while B only accepts `numpy.ndarray`. "NumPy or CuPy array" here is a superclass of NumPy array. Alternatively, if A accepts all subclasses of `numpy.ndarray` and B accepts only @@ -308,19 +323,20 @@ exactly `numpy.ndarray` (which spatch supports), then A is again the "superclass" because `numpy.ndarray` is also a subclasses of `numpy.ndarray` so A accepts a broader class of inputs. -In practice, the situation is even more complex, though. It is possible that +In practice, the situation is even more complex, though. It is possible that neither backend represents a clear superclass of the other and the above fails to establish an order. So we try to do the above, because we think it simplifies life of backend authors and ensure the correct type-dispatching order in some cases. -But of course it is not perfect! We think it would be a disservice to users to +But of course it is not perfect! We think it would be a disservice to users to attempt a more precise solution (no solution is perfect), because we want to provide users with an understandable, ordered, list of backends, but: -* We can't import all types for correct `issubclass` checks and we don't want the + +- We can't import all types for correct `issubclass` checks and we don't want the priority order to change if types get imported later. -* An extremely precise order might even consider types passed by the user but +- An extremely precise order might even consider types passed by the user but a context dependent order would be too hard to understand. So we infer the correct "type order" where it seems clear/simple enough and otherwise @@ -333,14 +349,14 @@ if it matters. We believe that the current complexity in spatch is manageable, although we would agree that a library that isn't dedicated to dispatching should likely avoid it. - ### Choice of environment variables -We don't mind changing these at all. There are three because: -* We need `_BLOCK` to allow avoiding loading a buggy backend entirely. +We don't mind changing these at all. There are three because: + +- We need `_BLOCK` to allow avoiding loading a buggy backend entirely. It is named "block" just because "disable" also makes sense at runtime. -* `_PRIORITIZE` is there as the main user API. -* `_SET_ORDER` is fine grained to prioritize one backend over another. +- `_PRIORITIZE` is there as the main user API. +- `_SET_ORDER` is fine grained to prioritize one backend over another. At runtime this seemed less important (can be done via introspection). This largely exists because if backend ordering is buggy, we tell users to set this to work around the issue. diff --git a/src/spatch/_spatch_example/README.md b/src/spatch/_spatch_example/README.md index a7a5bdc..94eb622 100644 --- a/src/spatch/_spatch_example/README.md +++ b/src/spatch/_spatch_example/README.md @@ -1,6 +1,6 @@ # Minimal usage example -A minimal example can be found below. If you are interested in the +A minimal example can be found below. If you are interested in the implementation side of this, please check [the source](https://github.com/scientific-python/spatch/spatch/_spatch_example). @@ -8,17 +8,19 @@ This is a very minimalistic example to show some of the concepts of creating a library and backends and how a user might work with them. The "library" contains only: -* `backend_opts()` a context manager for the user to change dispatching. -* A `divide` function that is dispatching enabled and assumed to be + +- `backend_opts()` a context manager for the user to change dispatching. +- A `divide` function that is dispatching enabled and assumed to be designed for only `int` inputs. We then have two backends with their corresponding definitions in `backend.py`. The entry-points are `entry_point.py` and `entry_point2.py` and these files can be run to generate their `functions` context (i.e. if you add more functions). -For users we have the following basic capabilities. Starting with normal +For users we have the following basic capabilities. Starting with normal type dispatching. First, import the functions and set up tracing globally: + ```pycon >>> import pprint >>> from spatch._spatch_example.library import divide, backend_opts @@ -26,7 +28,9 @@ First, import the functions and set up tracing globally: >>> opts.enable_globally() # or with opts() as trace: ``` + Now try calling the various implementations: + ```pycon >>> divide(1, 2) # use the normal library implementation (int inputs) 0 @@ -50,6 +54,7 @@ The first thing is to prioritize the use of a backend over another Backend 1 also has integers as a primary type, so we can prefer it over the default implementation for integer inputs as well: + ```pycon >>> with backend_opts(prioritize="backend1"): ... divide(1, 2) # now uses backend1 @@ -57,8 +62,10 @@ hello from backend 1 0 ``` + Similarly backend 2 supports floats, so we can prefer it over backend 1. We can still also prioritize "backend1" if we want: + ```pycon >>> with backend_opts(prioritize=["backend2", "backend1"]): ... divide(1., 2.) # now uses backend2 @@ -69,25 +76,28 @@ hello from backend 2 ('spatch._spatch_example.library:divide', [('backend2', 'called')])] ``` + The default priorities are based on the backend types or an explicit request to have a higher priority by a backend (otherwise default first and then alphabetically). Backends do have to make sure that the priorities make sense (i.e. there are no priority circles). -Prioritizing a backend will often effectively enable it. If such a backend changes +Prioritizing a backend will often effectively enable it. If such a backend changes behavior (e.g. faster but less precision) this can change results and confuse third party library functions. This is a user worry, backends must make sure that they never change types (even if prioritized), though. In the array world there use-cases that are not covered in the above: -* There are functions that create new arrays (say random number generators) - without inputs. We may wish to change their behavior within a scope or + +- There are functions that create new arrays (say random number generators) + without inputs. We may wish to change their behavior within a scope or globally. -* A user may try to bluntly modify behavior to use e.g. arrays on the GPU. +- A user may try to bluntly modify behavior to use e.g. arrays on the GPU. This is supported, but requires indicating the _type_ preference and users must be aware that this can even easier break their or third party code: + ```pycon >>> with backend_opts(type=float): ... divide(1, 2) # returns float (via backend 1) @@ -110,6 +120,7 @@ hello from backend 2 ('spatch._spatch_example.library:divide', [('backend2', 'called')])] ``` + How types work precisely should be decided by the backend, but care should be taken. E.g. it is not clear if returning a float is OK when the user said `type=complex`. (In the future, we may want to think more about this, especially if `type=complex|real` From 3b454a68d172b10cefd0d792e8e28aab77776d8e Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 14:51:54 -0500 Subject: [PATCH 17/33] sphinx-lint --- .pre-commit-config.yaml | 5 +++++ docs/source/api/for_backends.rst | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ca17db..9287e99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,6 +84,11 @@ repos: rev: v3.6.2 hooks: - id: prettier + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint + args: [--enable, all, "--disable=line-too-long,leaked-markup"] - repo: https://github.com/ComPWA/taplo-pre-commit rev: v0.9.3 hooks: diff --git a/docs/source/api/for_backends.rst b/docs/source/api/for_backends.rst index 128a8a8..fc59e3f 100644 --- a/docs/source/api/for_backends.rst +++ b/docs/source/api/for_backends.rst @@ -26,7 +26,7 @@ Before writing a backend, you need to think about a few things: by the user. Please check the example linked above. These example entry-points include -code that means running them modifies them in-place if the `@implements` +code that means running them modifies them in-place if the ``@implements`` decorator is used (see next section). Some of the most important things are: From 6f7968538af8275e8808b6d02e7702f06e1f1eeb Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 15:03:24 -0500 Subject: [PATCH 18/33] check-jsonschema and dependabot.yml --- .github/dependabot.yml | 11 +++++++++++ .pre-commit-config.yaml | 5 +++++ 2 files changed, 16 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9ecc51d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + groups: + actions: + patterns: + - "*" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9287e99..bfef157 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -97,6 +97,11 @@ repos: rev: v1.37.1 hooks: - id: yamllint + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.33.2 + hooks: + - id: check-dependabot + - id: check-github-workflows - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: From fb6c5d94b7bd71f2ca3cdf029e3e8907be5cd7fc Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 15:09:29 -0500 Subject: [PATCH 19/33] zizmor (TODO) --- .github/workflows/build-docs.yml | 1 + .github/workflows/ci.yml | 1 + .pre-commit-config.yaml | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 56af3c1..0c376df 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -32,6 +32,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e920e39..23fd00d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@v5 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bfef157..30ee742 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -102,6 +102,11 @@ repos: hooks: - id: check-dependabot - id: check-github-workflows + # TODO: get zizmor to pass, and maybe set it up as a github action + # - repo: https://github.com/woodruffw/zizmor-pre-commit + # rev: v1.11.0 + # hooks: + # - id: zizmor - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: From 0e91f36be90cb8221dfd27bd99d62cb65135321a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 15:21:09 -0500 Subject: [PATCH 20/33] blacken-docs --- .pre-commit-config.yaml | 11 ++++++++++- pyproject.toml | 1 - src/spatch/_spatch_example/README.md | 15 ++++++++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 30ee742..16b417e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,10 @@ # # $ pre-commit install # +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + autoupdate_schedule: "quarterly" fail_fast: false default_language_version: python: python3 @@ -54,10 +58,15 @@ repos: - id: pyupgrade args: [--py310-plus] # black often looks better than ruff-format - - repo: https://github.com/psf/black + - repo: https://github.com/psf/black-pre-commit-mirror rev: 25.1.0 hooks: - id: black + - repo: https://github.com/adamchainz/blacken-docs + rev: 1.19.1 + hooks: + - id: blacken-docs + additional_dependencies: [black==25.1.0] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.4 hooks: diff --git a/pyproject.toml b/pyproject.toml index 94aed9a..758493c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,6 @@ target-version = ["py310", "py311", "py312", "py313"] [tool.ruff] line-length = 100 -target-version = "py310" [tool.ruff.lint] extend-select = [ diff --git a/src/spatch/_spatch_example/README.md b/src/spatch/_spatch_example/README.md index 94eb622..68429c1 100644 --- a/src/spatch/_spatch_example/README.md +++ b/src/spatch/_spatch_example/README.md @@ -34,10 +34,10 @@ Now try calling the various implementations: ```pycon >>> divide(1, 2) # use the normal library implementation (int inputs) 0 ->>> divide(1., 2.) # uses backend 1 (float input) +>>> divide(1.0, 2.0) # uses backend 1 (float input) hello from backend 1 0.5 ->>> divide(1j, 2.) # uses backend 2 (complex input) +>>> divide(1j, 2.0) # uses backend 2 (complex input) hello from backend 2 0.5j >>> pprint.pprint(opts.trace) @@ -58,6 +58,7 @@ it over the default implementation for integer inputs as well: ```pycon >>> with backend_opts(prioritize="backend1"): ... divide(1, 2) # now uses backend1 +... hello from backend 1 0 @@ -68,7 +69,8 @@ We can still also prioritize "backend1" if we want: ```pycon >>> with backend_opts(prioritize=["backend2", "backend1"]): -... divide(1., 2.) # now uses backend2 +... divide(1.0, 2.0) # now uses backend2 +... hello from backend 2 0.5 >>> pprint.pprint(opts.trace[-2:]) @@ -101,17 +103,20 @@ must be aware that this can even easier break their or third party code: ```pycon >>> with backend_opts(type=float): ... divide(1, 2) # returns float (via backend 1) +... hello from backend 1 0.5 >>> with backend_opts(type=complex): ... # backen 2 returning a float for complex "input" is probably OK ... # (but may be debatable) ... divide(1, 2) +... hello from backend 2 0.5 >>> with backend_opts(type=float, prioritize="backend2"): -... # we can of course combine both type and prioritize. -... divide(1, 2) # backend 2 with float result (int inputs). +... # we can of course combine both type and prioritize. +... divide(1, 2) # backend 2 with float result (int inputs). +... hello from backend 2 0.5 >>> pprint.pprint(opts.trace[-3:]) From d8fa189d4099faebbb071a981e44ae33946bf8e7 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 15:32:40 -0500 Subject: [PATCH 21/33] pygrep-hooks and updates from sp-repo-review --- .pre-commit-config.yaml | 11 +++++++++++ pyproject.toml | 11 ++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16b417e..96d4e3e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -98,6 +98,17 @@ repos: hooks: - id: sphinx-lint args: [--enable, all, "--disable=line-too-long,leaked-markup"] + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-no-eval + - id: python-no-log-warn + - id: text-unicode-replacement-char - repo: https://github.com/ComPWA/taplo-pre-commit rev: v0.9.3 hooks: diff --git a/pyproject.toml b/pyproject.toml index 758493c..5f5ebf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,14 +66,23 @@ backend1 = 'spatch._spatch_example.entry_point' backend2 = 'spatch._spatch_example.entry_point2' [tool.pytest.ini_options] +minversion = "6.0" doctest_plus = "enabled" testpaths = [ "tests", "src/spatch", # for doc testing "docs", ] +xfail_strict = true norecursedirs = ["src"] -addopts = ["--doctest-glob=docs/source/**.md"] +addopts = [ + "--doctest-glob=docs/source/**.md", + "--strict-config", # Force error if config is misspelled + "--strict-markers", # Force error if marker is misspelled (must be defined in config) + "-ra", # Print summary of all fails/errors +] +log_cli_level = "info" +filterwarnings = ["error"] [tool.black] line-length = 100 From 750f5668f93dd117ff22639c65877e6d1bce8254 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 15:51:10 -0500 Subject: [PATCH 22/33] Add some local pre-commit hooks --- .gitignore | 1 + .pre-commit-config.yaml | 24 ++++++++++++++++++++++-- src/spatch/utils.py | 3 +-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index b97ecc8..54a1b43 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,4 @@ Thumbs.db # Common editor files *~ *.swp +*.swo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96d4e3e..4d8f9d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,9 +5,11 @@ # $ pre-commit install # ci: + autofix_prs: false + autoupdate_schedule: quarterly autoupdate_commit_msg: "chore: update pre-commit hooks" autofix_commit_msg: "style: pre-commit fixes" - autoupdate_schedule: "quarterly" + skip: [no-commit-to-branch] fail_fast: false default_language_version: python: python3 @@ -77,11 +79,13 @@ repos: hooks: - id: codespell types_or: [python, markdown, rst, toml, yaml] - additional_dependencies: [tomli] + additional_dependencies: + - tomli; python_version<'3.11' - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.4 hooks: - id: ruff-check + # - id: ruff-format # Prefer black, but may temporarily uncomment this to see # `pyroma` may help keep our package standards up to date if best practices change. # This is probably a "low value" check though and safe to remove if we want faster pre-commit. # - repo: https://github.com/regebro/pyroma @@ -127,6 +131,22 @@ repos: # rev: v1.11.0 # hooks: # - id: zizmor + - repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|Github|PyTest|RST|PyLint + exclude: (.pre-commit-config.yaml|docs/pages/guides/style\.md)$ + - id: disallow-words + name: Disallow certain words + language: pygrep + entry: "[Ff]alsey" + exclude: .pre-commit-config.yaml$ + - id: disallow-bad-permalinks + name: Disallow _ in permalinks + language: pygrep + entry: "^permalink:.*_.*" - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/src/spatch/utils.py b/src/spatch/utils.py index 8996080..b8b863c 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -45,8 +45,7 @@ def get_project_version(project_name, *, action_if_not_found="warn", default=Non """ if action_if_not_found not in {"ignore", "warn", "raise"}: raise ValueError( - "`action=` keyword must be 'ignore', 'warn', or 'raise'; " - f"got: {action_if_not_found!r}." + f"`action=` keyword must be 'ignore', 'warn', or 'raise'; got: {action_if_not_found!r}." ) try: project_version = version(project_name) From f4084474719870a39b950e07835def7e356e592f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 15:58:06 -0500 Subject: [PATCH 23/33] Add pre-commit github action --- .github/workflows/pre-commit.yml | 20 ++++++++++++++++++++ .pre-commit-config.yaml | 1 + 2 files changed, 21 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..4b51d39 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,20 @@ +name: pre-commit checks + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + name: pre-commit-hooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d8f9d4..b577110 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -97,6 +97,7 @@ repos: rev: v3.6.2 hooks: - id: prettier + args: [--prose-wrap=preserve] - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 hooks: From cf3e06f49205d19ceb8f971758b505b9942c33e0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 16:11:24 -0500 Subject: [PATCH 24/33] Clean up based on actual scientific-python cookie cutter template --- pyproject.toml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5f5ebf9..faa8f77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,22 +94,24 @@ line-length = 100 [tool.ruff.lint] extend-select = [ # Defaults from https://github.com/scientific-python/cookie excluding ARG and EM - "B", # flake8-bugbear - "I", # isort - "C4", # flake8-comprehensions - "ICN", # flake8-import-conventions - "PGH", # pygrep-hooks - "PIE", # flake8-pie - "PL", # pylint - "PT", # flake8-pytest-style - "PTH", # flake8-use-pathlib - "RET", # flake8-return - "RUF", # Ruff-specific - "SIM", # flake8-simplify - "TID251", # flake8-tidy-imports.banned-api - "T20", # flake8-print - "UP", # pyupgrade - "YTT", # flake8-2020 + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "EXE", # flake8-executable + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "NPY", # NumPy-specific rules + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "T20", # flake8-print + "UP", # pyupgrade + "YTT", # flake8-2020 # Additional ones (may be unnecessary, low value, or a nuisance). # It's okay to experiment and add or remove checks from here @@ -118,13 +120,10 @@ extend-select = [ "COM", # flake8-commas "DTZ", # flake8-datetimez "T10", # flake8-debugger - "EXE", # flake8-executable "ISC", # flake8-implicit-str-concat - "G", # flake8-logging-format "INP", # flake8-no-pep420 "Q", # flake8-quotes "RSE", # flake8-raise - "NPY", # NumPy-specific rules "N", # pep8-naming "PLC", # pylint Convention "PLE", # pylint Error @@ -190,6 +189,7 @@ ignore = [ "SIM102", # Use a single `if` statement instead of nested `if` statements (Note: often necessary) "SIM105", # Use contextlib.suppress(...) instead of try-except-pass (Note: try-except-pass is much faster) "SIM108", # Use ternary operator ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) + "TID251", # flake8-tidy-imports.banned-api "TRY003", # Avoid specifying long messages outside the exception class (Note: why?) "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` (Note: using `|` is slower atm) From 5af01bdc4271d6bc7b064455786f7732f37ac426 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 18:43:41 -0500 Subject: [PATCH 25/33] Fix docs (oops!) --- docs/source/design_choices.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/design_choices.md b/docs/source/design_choices.md index a219274..25fe5f9 100644 --- a/docs/source/design_choices.md +++ b/docs/source/design_choices.md @@ -25,9 +25,8 @@ Particularly important design considerations are: of library authors much harder. % This section really just to keep the first TOC a bit cleaner... -Specific design questions/reasons ---- +## Specific design questions/reasons The following are some specific questions/arguments. The answers may be rambling at times :). From c6a21fb92d95e884890793a3546e90ec9109210f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 18:44:20 -0500 Subject: [PATCH 26/33] Use `importlib_metadata.version` instead of `importlib.metadata.version`. --- src/spatch/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/spatch/utils.py b/src/spatch/utils.py index b8b863c..cf0b916 100644 --- a/src/spatch/utils.py +++ b/src/spatch/utils.py @@ -3,7 +3,8 @@ import warnings from dataclasses import dataclass, field from importlib import import_module -from importlib.metadata import version + +from importlib_metadata import version def get_identifier(obj): @@ -20,12 +21,12 @@ def from_identifier(ident): def get_project_version(project_name, *, action_if_not_found="warn", default=None): - """Get the version of a project from ``importlib.metadata``. + """Get the version of a project from ``importlib_metadata``. This is useful to ensure a package is properly installed regardless of the tools used to build the project and create the version. Proper installation is important to ensure entry-points of the project are discoverable. If the - project is not found by ``importlib.metadata``, behavior is controlled by + project is not found by ``importlib_metadata``, behavior is controlled by the ``action_if_not_found`` and ``default`` keyword arguments. Parameters From a0959410659a252c5f6a383b0fd262f2864fbbb4 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 18:54:14 -0500 Subject: [PATCH 27/33] Add `pre-commit` to `dev` dependency group --- pyproject.toml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index faa8f77..5593c38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", "hatch-vcs", "hatch-fancy-pypi-readme"] +requires = ["hatchling >=1.26", "hatch-vcs", "hatch-fancy-pypi-readme"] build-backend = "hatchling.build" [project] @@ -44,12 +44,16 @@ test = [ "pytest-doctestplus", { include-group = "backend_utils" }, ] -dev = [{ include-group = "test" }, { include-group = "docs" }] +dev = [ + "pre-commit >=4.1", + { include-group = "test" }, + { include-group = "docs" }, +] docs = [ - "sphinx>=7.0", + "sphinx >=7.0", "sphinx-copybutton", "pydata-sphinx-theme", - "myst-parser", + "myst-parser >=0.13", ] [tool.hatch.version] From 3bbab2eb2f4e386c8e617d5ce29239afee99f708 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 23 Jul 2025 22:44:16 -0500 Subject: [PATCH 28/33] auto-walrus (hehehe) --- .pre-commit-config.yaml | 5 +++++ docs/source/conf.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b577110..69cd760 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,6 +81,11 @@ repos: types_or: [python, markdown, rst, toml, yaml] additional_dependencies: - tomli; python_version<'3.11' + - repo: https://github.com/MarcoGorelli/auto-walrus + rev: 0.3.4 + hooks: + - id: auto-walrus + args: [--line-length, "100"] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.4 hooks: diff --git a/docs/source/conf.py b/docs/source/conf.py index 476d644..3004ac9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,11 +1,11 @@ from __future__ import annotations -import importlib.metadata +import importlib_metadata project = "spatch" copyright = "2025, Spatch authors" author = "Spatch authors" -version = release = importlib.metadata.version("spatch") +version = release = importlib_metadata.version("spatch") extensions = [ "myst_parser", From b79a8fac1c644f9de8732dc3d233c68d39c55aa0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 24 Jul 2025 10:24:27 -0500 Subject: [PATCH 29/33] Use `Path(filepath).read_text()` Co-authored-by: Sebastian Berg --- src/spatch/backend_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index 70a1035..d4066c2 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -235,8 +235,7 @@ def update_entrypoint( ] # Step 4: replace text - with Path(filepath).open("r") as f: - text = f.read() + text = Path(filepath).read_text() text = update_text(text, lines, "functions", indent=indent) with Path(filepath).open("w") as f: f.write(text) From cdb1ee8876bf8eaca763ae9ecfe0bc1635ac9734 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 24 Jul 2025 10:25:01 -0500 Subject: [PATCH 30/33] Use `Path(filepath).write_text(text)` Co-authored-by: Sebastian Berg --- src/spatch/backend_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index d4066c2..5ca0577 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -237,5 +237,4 @@ def update_entrypoint( # Step 4: replace text text = Path(filepath).read_text() text = update_text(text, lines, "functions", indent=indent) - with Path(filepath).open("w") as f: - f.write(text) + Path(filepath).write_text(text) From e20f2602d0e97ecff5e2b38b17c0f079409831d0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 24 Jul 2025 09:34:39 -0500 Subject: [PATCH 31/33] oops fix whitespace --- src/spatch/backend_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spatch/backend_utils.py b/src/spatch/backend_utils.py index 5ca0577..4820d14 100644 --- a/src/spatch/backend_utils.py +++ b/src/spatch/backend_utils.py @@ -237,4 +237,4 @@ def update_entrypoint( # Step 4: replace text text = Path(filepath).read_text() text = update_text(text, lines, "functions", indent=indent) - Path(filepath).write_text(text) + Path(filepath).write_text(text) From 5f56a61d56cb961d3197ca7223b552c4560ceb36 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 24 Jul 2025 10:29:53 -0500 Subject: [PATCH 32/33] Iterate over `graph` since not mutating Co-authored-by: Sebastian Berg --- src/spatch/backend_system.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/spatch/backend_system.py b/src/spatch/backend_system.py index 2598889..a98b8f9 100644 --- a/src/spatch/backend_system.py +++ b/src/spatch/backend_system.py @@ -568,9 +568,8 @@ def visit(node, order, _visiting): del _visiting[node] order[node] = None # add sorted node - # to_sort = list(graph.keys()) # @eriknw to @seberg: should this be used? order = {} # dict as a sorted set - for n in list(graph.keys()): + for n in graph: visit(n, order, {}) return tuple(order.keys()) From b0ae0827ae45a80aa35aac45c89faa8dad852025 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 24 Jul 2025 09:39:19 -0500 Subject: [PATCH 33/33] bump ruff --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69cd760..22fe6c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -70,7 +70,7 @@ repos: - id: blacken-docs additional_dependencies: [black==25.1.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.4 + rev: v0.12.5 hooks: - id: ruff-check args: [--fix-only, --show-fixes] @@ -87,7 +87,7 @@ repos: - id: auto-walrus args: [--line-length, "100"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.4 + rev: v0.12.5 hooks: - id: ruff-check # - id: ruff-format # Prefer black, but may temporarily uncomment this to see