[pull] main from python:main

Doc/library/contextlib.rst

@@ -467,12 +467,40 @@ Functions and classes provided:
       statements. If this is not the case, then the original construct with the
       explicit :keyword:`!with` statement inside the function should be used.
 
+   When the decorated callable is a generator function, coroutine function, or
+   asynchronous generator function, the returned wrapper is of the same kind
+   and keeps the context manager open for the lifetime of the iteration or
+   await rather than only for the call that creates the generator or coroutine
+   object.  Wrapped generators and asynchronous generators are explicitly
+   closed when iteration ends, as if by :func:`closing` or :func:`aclosing`.
+
+   .. note::
+      For asynchronous generators the wrapper re-yields each value with
+      ``async for``; values sent with :meth:`~agen.asend` and exceptions
+      thrown with :meth:`~agen.athrow` are not forwarded to the wrapped
+      generator.
+
    .. versionadded:: 3.2
 
+   .. versionchanged:: next
+      Decorating a generator function, coroutine function, or asynchronous
+      generator function now keeps the context manager open across iteration
+      or await.  Previously the context manager exited as soon as the
+      generator or coroutine object was created.
+
 
 .. class:: AsyncContextDecorator
 
-   Similar to :class:`ContextDecorator` but only for asynchronous functions.
+   Similar to :class:`ContextDecorator`, but the context manager is entered
+   and exited with :keyword:`async with`.  Decorate coroutine functions and
+   asynchronous generator functions with this class; the returned wrapper is
+   of the same kind.
+
+   .. note::
+      Synchronous functions and generators are accepted, but the wrapper is
+      always asynchronous, so the decorated callable must then be awaited or
+      iterated with ``async for``.  If that change of calling convention is
+      not intended, use :class:`ContextDecorator` instead.
 
    Example of ``AsyncContextDecorator``::
 
@@ -510,6 +538,13 @@ Functions and classes provided:
 
    .. versionadded:: 3.10
 
+   .. versionchanged:: next
+      Decorating an asynchronous generator function now keeps the context
+      manager open across iteration.  Previously the context manager exited
+      as soon as the generator object was created.  Synchronous functions
+      and synchronous generator functions are also now accepted, with an
+      asynchronous wrapper returned.
+
 
 .. class:: ExitStack()
 

Doc/whatsnew/3.15.rst

@@ -677,6 +677,13 @@ Other language changes
   the existing support for unary minus.
   (Contributed by Bartosz Sławecki in :gh:`145239`.)
 
+* The import system now acquires per-module locks in hierarchical order
+  (parent packages before their submodules). This fixes a long-standing
+  deadlock where one thread importing ``pkg.sub`` and another importing
+  ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py``
+  imports ``pkg.sub.mod``.
+  (Contributed by Gregory P. Smith in :gh:`83065`.)
+
 
 New modules
 ===========
@@ -846,6 +853,15 @@ contextlib
   consistency with the :keyword:`with` and :keyword:`async with` statements.
   (Contributed by Serhiy Storchaka in :gh:`144386`.)
 
+* :class:`~contextlib.ContextDecorator` and
+  :class:`~contextlib.AsyncContextDecorator` (and therefore
+  :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager`
+  used as decorators) now detect generator functions, coroutine functions, and
+  asynchronous generator functions and keep the context manager open across
+  iteration or await.  Previously the context manager exited as soon as the
+  generator or coroutine object was created.
+  (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.)
+
 
 dataclasses
 -----------

Lib/contextlib.py

@@ -1,10 +1,16 @@
 """Utilities for with-statement contexts.  See PEP 343."""
+
 import abc
 import os
 import sys
 import _collections_abc
 from collections import deque
 from functools import wraps
+lazy from inspect import (
+    isasyncgenfunction as _isasyncgenfunction,
+    iscoroutinefunction as _iscoroutinefunction,
+    isgeneratorfunction as _isgeneratorfunction,
+)
 from types import GenericAlias
 
 __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
@@ -79,11 +85,37 @@ def _recreate_cm(self):
         return self
 
     def __call__(self, func):
-        @wraps(func)
-        def inner(*args, **kwds):
-            with self._recreate_cm():
-                return func(*args, **kwds)
-        return inner
+        wrapper = wraps(func)
+        if _isasyncgenfunction(func):
+
+            async def asyncgen_inner(*args, **kwds):
+                with self._recreate_cm():
+                    async with aclosing(func(*args, **kwds)) as gen:
+                        async for value in gen:
+                            yield value
+
+            return wrapper(asyncgen_inner)
+        elif _iscoroutinefunction(func):
+
+            async def async_inner(*args, **kwds):
+                with self._recreate_cm():
+                    return await func(*args, **kwds)
+
+            return wrapper(async_inner)
+        elif _isgeneratorfunction(func):
+
+            def gen_inner(*args, **kwds):
+                with self._recreate_cm(), closing(func(*args, **kwds)) as gen:
+                    return (yield from gen)
+
+            return wrapper(gen_inner)
+        else:
+
+            def inner(*args, **kwds):
+                with self._recreate_cm():
+                    return func(*args, **kwds)
+
+            return wrapper(inner)
 
 
 class AsyncContextDecorator(object):
@@ -95,11 +127,41 @@ def _recreate_cm(self):
         return self
 
     def __call__(self, func):
-        @wraps(func)
-        async def inner(*args, **kwds):
-            async with self._recreate_cm():
-                return await func(*args, **kwds)
-        return inner
+        wrapper = wraps(func)
+        if _isasyncgenfunction(func):
+
+            async def asyncgen_inner(*args, **kwds):
+                async with (
+                    self._recreate_cm(),
+                    aclosing(func(*args, **kwds)) as gen
+                ):
+                    async for value in gen:
+                        yield value
+
+            return wrapper(asyncgen_inner)
+        elif _iscoroutinefunction(func):
+
+            async def async_inner(*args, **kwds):
+                async with self._recreate_cm():
+                    return await func(*args, **kwds)
+
+            return wrapper(async_inner)
+        elif _isgeneratorfunction(func):
+
+            async def gen_inner(*args, **kwds):
+                async with self._recreate_cm():
+                    with closing(func(*args, **kwds)) as gen:
+                        for value in gen:
+                            yield value
+
+            return wrapper(gen_inner)
+        else:
+
+            async def inner(*args, **kwds):
+                async with self._recreate_cm():
+                    return func(*args, **kwds)
+
+            return wrapper(inner)
 
 
 class _GeneratorContextManagerBase:

Lib/importlib/_bootstrap.py

@@ -424,6 +424,64 @@ def __exit__(self, *args, **kwargs):
         self._lock.release()
 
 
+def _get_module_chain(name):
+    """Return the chain of dotted-name prefixes from root to leaf.
+
+    For example: 'a.b.c' -> ['a', 'a.b', 'a.b.c']
+    """
+    parts = name.split('.')
+    return ['.'.join(parts[:i+1]) for i in range(len(parts))]
+
+
+class _HierarchicalLockManager:
+    """Manages acquisition of multiple module locks in hierarchical order.
+
+    This prevents deadlocks by ensuring all threads acquire locks in the
+    same order (parent modules before child modules).
+    """
+
+    def __init__(self, name):
+        self._name = name
+        self._module_chain = _get_module_chain(name)
+        self._locks = []
+
+    def __enter__(self):
+        try:
+            for module_name in self._module_chain:
+                # Only acquire lock if module is not already fully loaded
+                module = sys.modules.get(module_name)
+                if (module is None or
+                    getattr(getattr(module, "__spec__", None),
+                            "_initializing", False)):
+                    lock = _get_module_lock(module_name)
+                    try:
+                        lock.acquire()
+                    except _DeadlockError:
+                        if module_name == self._name:
+                            raise
+                        # The parent is being initialised by a thread that
+                        # is (transitively) waiting on a lock we hold.
+                        # Apply the same policy as _lock_unlock_module():
+                        # accept a partially-initialised parent for circular
+                        # imports rather than failing the whole chain.
+                        continue
+                    self._locks.append((module_name, lock))
+        except:
+            # __exit__ is not called when __enter__ raises (e.g. _DeadlockError
+            # on the leaf lock, or KeyboardInterrupt), so release whatever we
+            # already hold to avoid permanently leaking held module locks.
+            for module_name, lock in reversed(self._locks):
+                lock.release()
+            self._locks.clear()
+            raise
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        for module_name, lock in reversed(self._locks):
+            lock.release()
+        self._locks.clear()
+
+
 # The following two functions are for consumption by Python/import.c.
 
 def _get_module_lock(name):
@@ -1276,7 +1334,13 @@ def _find_and_load(name, import_):
     module = sys.modules.get(name, _NEEDS_LOADING)
     if (module is _NEEDS_LOADING or
         getattr(getattr(module, "__spec__", None), "_initializing", False)):
-        with _ModuleLockManager(name):
+
+        if '.' in name:
+            lock_manager = _HierarchicalLockManager(name)
+        else:
+            lock_manager = _ModuleLockManager(name)
+
+        with lock_manager:
             module = sys.modules.get(name, _NEEDS_LOADING)
             if module is _NEEDS_LOADING:
                 return _find_and_load_unlocked(name, import_)