From c07e5ec0a9e5843fc39dec6aa94172faf6354858 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 8 Jan 2026 13:51:38 +0200 Subject: [PATCH 1/7] gh-143553: Add support for parametrized resources in regrtests (GH-143554) For example, "-u xpickle=2.7" will run test_xpickle only against Python 2.7. --- Doc/library/test.rst | 6 ++ Lib/test/libregrtest/cmdline.py | 47 +++++----- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/runtests.py | 11 ++- Lib/test/libregrtest/utils.py | 27 ++++-- Lib/test/support/__init__.py | 15 +++- Lib/test/test_regrtest.py | 85 +++++++++++++------ Lib/test/test_xpickle.py | 14 +-- ...-01-08-11-50-06.gh-issue-143553.KyyNTt.rst | 1 + 9 files changed, 139 insertions(+), 69 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-01-08-11-50-06.gh-issue-143553.KyyNTt.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 395cde21ccf449..44b1d395a27d13 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -492,6 +492,12 @@ The :mod:`test.support` module defines the following functions: tests. +.. function:: get_resource_value(resource) + + Return the value specified for *resource* (as :samp:`-u {resource}={value}`). + Return ``None`` if *resource* is disabled or no value is specified. + + .. function:: python_is_optimized() Return ``True`` if Python was not built with ``-O0`` or ``-Og``. diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 42edb73496c752..d784506703461b 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -162,7 +162,7 @@ def __init__(self, **kwargs) -> None: self.randomize = False self.fromfile = None self.fail_env_changed = False - self.use_resources: list[str] = [] + self.use_resources: dict[str, str | None] = {} self.trace = False self.coverdir = 'coverage' self.runleaks = False @@ -309,7 +309,7 @@ def _create_parser(): group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, + action='extend', type=resources_list, help='specify which special resource intensive tests ' 'to run.' + more_details) group.add_argument('-M', '--memlimit', metavar='LIMIT', @@ -414,11 +414,18 @@ def huntrleaks(string): def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: + u = [] + for x in string.split(','): + r, eq, v = x.partition('=') + r = r.lower() + u.append((r, v if eq else None)) if r == 'all' or r == 'none': + if eq: + raise argparse.ArgumentTypeError('invalid resource: ' + x) continue if r[0] == '-': + if eq: + raise argparse.ArgumentTypeError('invalid resource: ' + x) r = r[1:] if r not in RESOURCE_NAMES: raise argparse.ArgumentTypeError('invalid resource: ' + r) @@ -486,14 +493,14 @@ def _parse_args(args, **kwargs): # Similar to: -u "all" --timeout=1200 if ns.use is None: ns.use = [] - ns.use.insert(0, ['all']) + ns.use[:0] = [('all', None)] if ns.timeout is None: ns.timeout = 1200 # 20 minutes elif ns.fast_ci: # Similar to: -u "all,-cpu" --timeout=600 if ns.use is None: ns.use = [] - ns.use.insert(0, ['all', '-cpu']) + ns.use[:0] = [('all', None), ('-cpu', None)] if ns.timeout is None: ns.timeout = 600 # 10 minutes @@ -531,23 +538,17 @@ def _parse_args(args, **kwargs): if ns.timeout <= 0: ns.timeout = None if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = ALL_RESOURCES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) + for r, v in ns.use: + if r == 'all': + for r in ALL_RESOURCES: + ns.use_resources[r] = None + elif r == 'none': + ns.use_resources.clear() + elif r[0] == '-': + r = r[1:] + ns.use_resources.pop(r, None) + else: + ns.use_resources[r] = v if ns.random_seed is not None: ns.randomize = True if ns.no_randomize: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0fc2548789e2e1..d8b9605ea49843 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -118,7 +118,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.junit_filename: StrPath | None = ns.xmlpath self.memory_limit: str | None = ns.memlimit self.gc_threshold: int | None = ns.threshold - self.use_resources: tuple[str, ...] = tuple(ns.use_resources) + self.use_resources: dict[str, str | None] = dict(ns.use_resources) if ns.python: self.python_cmd: tuple[str, ...] | None = tuple(ns.python) else: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 759f24fc25e38c..e6d34d8e6a3be5 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -96,7 +96,7 @@ class RunTests: coverage: bool memory_limit: str | None gc_threshold: int | None - use_resources: tuple[str, ...] + use_resources: dict[str, str | None] python_cmd: tuple[str, ...] | None randomize: bool random_seed: int | str @@ -179,7 +179,14 @@ def bisect_cmd_args(self) -> list[str]: if self.gc_threshold: args.append(f"--threshold={self.gc_threshold}") if self.use_resources: - args.extend(("-u", ','.join(self.use_resources))) + simple = ','.join(resource + for resource, value in self.use_resources.items() + if value is None) + if simple: + args.extend(("-u", simple)) + for resource, value in self.use_resources.items(): + if value is not None: + args.extend(("-u", f"{resource}={value}")) if self.python_cmd: cmd = shlex.join(self.python_cmd) args.extend(("--python", cmd)) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 1daa9c7baf8211..4479f336b1ee53 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -12,7 +12,7 @@ import sysconfig import tempfile import textwrap -from collections.abc import Callable, Iterable +from collections.abc import Callable from test import support from test.support import os_helper @@ -607,21 +607,30 @@ def is_cross_compiled() -> bool: return ('_PYTHON_HOST_PLATFORM' in os.environ) -def format_resources(use_resources: Iterable[str]) -> str: - use_resources = set(use_resources) +def format_resources(use_resources: dict[str, str | None]) -> str: all_resources = set(ALL_RESOURCES) + values = [] + for name in sorted(use_resources): + if use_resources[name] is not None: + values.append(f'{name}={use_resources[name]}') + # Express resources relative to "all" relative_all = ['all'] - for name in sorted(all_resources - use_resources): + for name in sorted(all_resources - set(use_resources)): relative_all.append(f'-{name}') - for name in sorted(use_resources - all_resources): - relative_all.append(f'{name}') - all_text = ','.join(relative_all) + for name in sorted(set(use_resources) - all_resources): + if use_resources[name] is None: + relative_all.append(name) + all_text = ','.join(relative_all + values) all_text = f"resources: {all_text}" # List of enabled resources - text = ','.join(sorted(use_resources)) + resources = [] + for name in sorted(use_resources): + if use_resources[name] is None: + resources.append(name) + text = ','.join(resources + values) text = f"resources ({len(use_resources)}): {text}" # Pick the shortest string (prefer relative to all if lengths are equal) @@ -631,7 +640,7 @@ def format_resources(use_resources: Iterable[str]) -> str: return text -def display_header(use_resources: tuple[str, ...], +def display_header(use_resources: dict[str, str | None], python_cmd: tuple[str, ...] | None) -> None: # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 84fd43fd396914..847d9074eb82cd 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -30,7 +30,8 @@ "record_original_stdout", "get_original_stdout", "captured_stdout", "captured_stdin", "captured_stderr", "captured_output", # unittest - "is_resource_enabled", "requires", "requires_freebsd_version", + "is_resource_enabled", "get_resource_value", "requires", "requires_resource", + "requires_freebsd_version", "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", "requires_gzip", "requires_bz2", "requires_lzma", "requires_zstd", @@ -185,7 +186,7 @@ def get_attribute(obj, name): return attribute verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # Flag set to [] by regrtest.py +use_resources = None # Flag set to {} by regrtest.py max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 @@ -300,6 +301,16 @@ def is_resource_enabled(resource): """ return use_resources is None or resource in use_resources +def get_resource_value(resource): + """Test whether a resource is enabled. + + Known resources are set by regrtest.py. If not running under regrtest.py, + all resources are assumed enabled unless use_resources has been set. + """ + if use_resources is None: + return None + return use_resources.get(resource) + def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" if not is_resource_enabled(resource): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c27b3c862924d1..fc6694d489fb0f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -279,26 +279,56 @@ def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): ns = self.parse_args([opt, 'gui,network']) - self.assertEqual(ns.use_resources, ['gui', 'network']) + self.assertEqual(ns.use_resources, {'gui': None, 'network': None}) + ns = self.parse_args([opt, 'gui', opt, 'network']) + self.assertEqual(ns.use_resources, {'gui': None, 'network': None}) ns = self.parse_args([opt, 'gui,none,network']) - self.assertEqual(ns.use_resources, ['network']) + self.assertEqual(ns.use_resources, {'network': None}) + ns = self.parse_args([opt, 'gui', opt, 'none', opt, 'network']) + self.assertEqual(ns.use_resources, {'network': None}) - expected = list(cmdline.ALL_RESOURCES) - expected.remove('gui') + expected = dict.fromkeys(cmdline.ALL_RESOURCES) + del expected['gui'] ns = self.parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) + self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') # all + a resource not part of "all" + expected = dict.fromkeys(cmdline.ALL_RESOURCES) + expected['tzdata'] = None ns = self.parse_args([opt, 'all,tzdata']) - self.assertEqual(ns.use_resources, - list(cmdline.ALL_RESOURCES) + ['tzdata']) + self.assertEqual(ns.use_resources, expected) + ns = self.parse_args([opt, 'all', opt, 'tzdata']) + self.assertEqual(ns.use_resources, expected) # test another resource which is not part of "all" ns = self.parse_args([opt, 'extralargefile']) - self.assertEqual(ns.use_resources, ['extralargefile']) + self.assertEqual(ns.use_resources, {'extralargefile': None}) + + # test resource with value + ns = self.parse_args([opt, 'xpickle=2.7']) + self.assertEqual(ns.use_resources, {'xpickle': '2.7'}) + ns = self.parse_args([opt, 'xpickle=2.7,xpickle=3.3']) + self.assertEqual(ns.use_resources, {'xpickle': '3.3'}) + ns = self.parse_args([opt, 'xpickle=2.7,none']) + self.assertEqual(ns.use_resources, {}) + ns = self.parse_args([opt, 'xpickle=2.7,-xpickle']) + self.assertEqual(ns.use_resources, {}) + + expected = dict.fromkeys(cmdline.ALL_RESOURCES) + expected['xpickle'] = '2.7' + ns = self.parse_args([opt, 'all,xpickle=2.7']) + self.assertEqual(ns.use_resources, expected) + ns = self.parse_args([opt, 'all', opt, 'xpickle=2.7']) + self.assertEqual(ns.use_resources, expected) + + # test invalid resources with value + self.checkError([opt, 'all=0'], 'invalid resource: all=0') + self.checkError([opt, 'none=0'], 'invalid resource: none=0') + self.checkError([opt, 'all,-gui=0'], 'invalid resource: -gui=0') def test_memlimit(self): for opt in '-M', '--memlimit': @@ -459,20 +489,20 @@ def check_ci_mode(self, args, use_resources, self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.print_slowest) self.assertEqual(regrtest.output_on_failure, output_on_failure) - self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) + self.assertEqual(regrtest.use_resources, use_resources) return regrtest def test_fast_ci(self): args = ['--fast-ci'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 10 * 60) def test_fast_ci_python_cmd(self): args = ['--fast-ci', '--python', 'python -X dev'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] regrtest = self.check_ci_mode(args, use_resources, rerun=False) self.assertEqual(regrtest.timeout, 10 * 60) self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev')) @@ -480,32 +510,33 @@ def test_fast_ci_python_cmd(self): def test_fast_ci_resource(self): # it should be possible to override resources individually args = ['--fast-ci', '-u-network'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') - use_resources.remove('network') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] + del use_resources['network'] self.check_ci_mode(args, use_resources) def test_fast_ci_verbose(self): args = ['--fast-ci', '--verbose'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] regrtest = self.check_ci_mode(args, use_resources, output_on_failure=False) self.assertEqual(regrtest.verbose, True) def test_slow_ci(self): args = ['--slow-ci'] - use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 20 * 60) def test_ci_no_randomize(self): - all_resources = set(cmdline.ALL_RESOURCES) + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) self.check_ci_mode( - ["--slow-ci", "--no-randomize"], all_resources, randomize=False + ["--slow-ci", "--no-randomize"], use_resources, randomize=False ) + del use_resources['cpu'] self.check_ci_mode( - ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False + ["--fast-ci", "--no-randomize"], use_resources, randomize=False ) def test_dont_add_python_opts(self): @@ -2435,20 +2466,20 @@ def test_format_resources(self): format_resources = utils.format_resources ALL_RESOURCES = utils.ALL_RESOURCES self.assertEqual( - format_resources(("network",)), + format_resources({"network": None}), 'resources (1): network') self.assertEqual( - format_resources(("audio", "decimal", "network")), + format_resources(dict.fromkeys(("audio", "decimal", "network"))), 'resources (3): audio,decimal,network') self.assertEqual( - format_resources(ALL_RESOURCES), + format_resources(dict.fromkeys(ALL_RESOURCES)), 'resources: all') self.assertEqual( - format_resources(tuple(name for name in ALL_RESOURCES - if name != "cpu")), + format_resources({name: None for name in ALL_RESOURCES + if name != "cpu"}), 'resources: all,-cpu') self.assertEqual( - format_resources((*ALL_RESOURCES, "tzdata")), + format_resources({**dict.fromkeys(ALL_RESOURCES), "tzdata": None}), 'resources: all,tzdata') def test_match_test(self): diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 659d3e38389860..158f27dce4fdc2 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -230,11 +230,15 @@ def add_tests(py_version): test_class = make_test(py_version, CPicklePythonCompat) tests.addTest(loader.loadTestsFromTestCase(test_class)) - major = sys.version_info.major - assert major == 3 - add_tests((2, 7)) - for minor in range(2, sys.version_info.minor): - add_tests((major, minor)) + value = support.get_resource_value('xpickle') + if value is None: + major = sys.version_info.major + assert major == 3 + add_tests((2, 7)) + for minor in range(2, sys.version_info.minor): + add_tests((major, minor)) + else: + add_tests(tuple(map(int, value.split('.')))) return tests diff --git a/Misc/NEWS.d/next/Tests/2026-01-08-11-50-06.gh-issue-143553.KyyNTt.rst b/Misc/NEWS.d/next/Tests/2026-01-08-11-50-06.gh-issue-143553.KyyNTt.rst new file mode 100644 index 00000000000000..e9509057601979 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-01-08-11-50-06.gh-issue-143553.KyyNTt.rst @@ -0,0 +1 @@ +Add support for parametrized resources, such as ``-u xpickle=2.7``. From f3e069a7ab8b0594508c998da88937e3aab46451 Mon Sep 17 00:00:00 2001 From: Aniket <148300120+Aniketsy@users.noreply.github.com> Date: Thu, 8 Jan 2026 18:10:25 +0530 Subject: [PATCH 2/7] gh-140025: Fix `queue.SimpleQueue.__sizeof__()` to return correct size (#143137) --- Lib/test/test_queue.py | 10 ++++++ ...-01-02-09-32-43.gh-issue-140025.zOX58_.rst | 1 + Modules/_queuemodule.c | 17 ++++++++++ Modules/clinic/_queuemodule.c.h | 32 ++++++++++++++++++- 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-01-02-09-32-43.gh-issue-140025.zOX58_.rst diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index c855fb8fe2b05a..f2898de469e349 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -2,6 +2,7 @@ # to ensure the Queue locks remain stable. import itertools import random +import struct import threading import time import unittest @@ -9,6 +10,7 @@ from test.support import gc_collect, bigmemtest from test.support import import_helper from test.support import threading_helper +from test import support # queue module depends on threading primitives threading_helper.requires_working_threading(module=True) @@ -1031,6 +1033,14 @@ def test_is_default(self): self.assertIs(self.type2test, self.queue.SimpleQueue) self.assertIs(self.type2test, self.queue.SimpleQueue) + def test_simplequeue_sizeof(self): + q = self.type2test() + basesize = support.calcobjsize('?nnPnnP') + support.check_sizeof(self, q, basesize + struct.calcsize(8 * 'P')) + for _ in range(1000): + q.put(object()) + support.check_sizeof(self, q, basesize + struct.calcsize(1024 * 'P')) + def test_reentrancy(self): # bpo-14976: put() may be called reentrantly in an asynchronous # callback. diff --git a/Misc/NEWS.d/next/Library/2026-01-02-09-32-43.gh-issue-140025.zOX58_.rst b/Misc/NEWS.d/next/Library/2026-01-02-09-32-43.gh-issue-140025.zOX58_.rst new file mode 100644 index 00000000000000..cb5458f28af667 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-02-09-32-43.gh-issue-140025.zOX58_.rst @@ -0,0 +1 @@ +:mod:`queue`: Fix :meth:`!SimpleQueue.__sizeof__` computation. diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 01235c77bd7db8..a45959346bc1f2 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -500,6 +500,22 @@ _queue_SimpleQueue_qsize_impl(simplequeueobject *self) return RingBuf_Len(&self->buf); } +/*[clinic input] +@critical_section +_queue.SimpleQueue.__sizeof__ -> Py_ssize_t + +Returns size in memory, in bytes. +[clinic start generated code]*/ + +static Py_ssize_t +_queue_SimpleQueue___sizeof___impl(simplequeueobject *self) +/*[clinic end generated code: output=58ce4e3bbc078fd4 input=a3a7f05c9616598f]*/ +{ + Py_ssize_t res = sizeof(simplequeueobject); + res += self->buf.items_cap * sizeof(PyObject *); + return res; +} + static int queue_traverse(PyObject *m, visitproc visit, void *arg) { @@ -534,6 +550,7 @@ static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_PUT_METHODDEF _QUEUE_SIMPLEQUEUE_PUT_NOWAIT_METHODDEF _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF + _QUEUE_SIMPLEQUEUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 1751d68716ba5f..c9482f40acb9d4 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -358,4 +358,34 @@ _queue_SimpleQueue_qsize(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=1d3efe9df89997cf input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_queue_SimpleQueue___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Returns size in memory, in bytes."); + +#define _QUEUE_SIMPLEQUEUE___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)_queue_SimpleQueue___sizeof__, METH_NOARGS, _queue_SimpleQueue___sizeof____doc__}, + +static Py_ssize_t +_queue_SimpleQueue___sizeof___impl(simplequeueobject *self); + +static PyObject * +_queue_SimpleQueue___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + Py_ssize_t _return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + _return_value = _queue_SimpleQueue___sizeof___impl((simplequeueobject *)self); + Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=4af5d1b1ea31ac7d input=a9049054013a1b77]*/ From efaa56f73cb1dc4863894444425e753afb7b997c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 8 Jan 2026 14:05:57 +0100 Subject: [PATCH 3/7] gh-143528: Fix test_time.test_thread_time() (#143558) Tolerate 100 ms instead of 20 ms to support slow CIs. --- Lib/test/test_time.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index c7e81fff6f776b..715aaf384a309c 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -580,11 +580,10 @@ def test_thread_time(self): # thread_time() should not include time spend during a sleep start = time.thread_time() - time.sleep(0.100) + time.sleep(0.200) stop = time.thread_time() - # use 20 ms because thread_time() has usually a resolution of 15 ms - # on Windows - self.assertLess(stop - start, 0.020) + # gh-143528: use 100 ms to support slow CI + self.assertLess(stop - start, 0.100) info = time.get_clock_info('thread_time') self.assertTrue(info.monotonic) From 5462002bbefd6cb79716828051beae94de8f2529 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 8 Jan 2026 13:40:17 +0000 Subject: [PATCH 4/7] gh-143513: Clarify changed argument name of `ResourceReader.is_resource` (GH-143523) --- Doc/library/importlib.resources.abc.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 8253a33f591a0b..45979c5691270a 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -63,11 +63,14 @@ If the resource does not concretely exist on the file system, raise :exc:`FileNotFoundError`. - .. method:: is_resource(name) + .. method:: is_resource(path) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *path* is considered a resource. + :exc:`FileNotFoundError` is raised if *path* does not exist. + + .. versionchanged:: 3.10 + The argument *name* was renamed to *path*. .. method:: contents() :abstractmethod: From 49c3b0a67a77bb42e736cea7dcbc1aa8fa704074 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 8 Jan 2026 09:18:24 -0500 Subject: [PATCH 5/7] gh-142095: Use thread local frame info in `py-bt` and `py-bt-full` when available (gh-143371) In optimized and `-Og` builds, arguments and local variables are frequently unavailable in gdb. This makes `py-bt` fail to print anything useful. Use the `PyThreadState*` pointers `_Py_tss_gilstate` and `Py_tss_tstate` to find the interpreter frame if we can't get the frame from the `_PyEval_EvalFrameDefault` call. Co-authored-by: Victor Stinner --- ...-01-02-11-44-56.gh-issue-142095.4ssgnM.rst | 2 + Tools/gdb/libpython.py | 183 ++++++++++-------- 2 files changed, 109 insertions(+), 76 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-01-02-11-44-56.gh-issue-142095.4ssgnM.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-01-02-11-44-56.gh-issue-142095.4ssgnM.rst b/Misc/NEWS.d/next/Tools-Demos/2026-01-02-11-44-56.gh-issue-142095.4ssgnM.rst new file mode 100644 index 00000000000000..196b27dfd66302 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-01-02-11-44-56.gh-issue-142095.4ssgnM.rst @@ -0,0 +1,2 @@ +Make gdb 'py-bt' command use frame from thread local state when available. +Patch by Sam Gross and Victor Stinner. diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 27aa6b0cc266d3..a85195dcd1016a 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -152,6 +152,11 @@ def write(self, data): def getvalue(self): return self._val + +def _PyStackRef_AsPyObjectBorrow(gdbval): + return gdb.Value(int(gdbval['bits']) & ~USED_TAGS) + + class PyObjectPtr(object): """ Class wrapping a gdb.Value that's either a (PyObject*) within the @@ -170,7 +175,7 @@ def __init__(self, gdbval, cast_to=None): if gdbval.type.name == '_PyStackRef': if cast_to is None: cast_to = gdb.lookup_type('PyObject').pointer() - self._gdbval = gdb.Value(int(gdbval['bits']) & ~USED_TAGS).cast(cast_to) + self._gdbval = _PyStackRef_AsPyObjectBorrow(gdbval).cast(cast_to) elif cast_to: self._gdbval = gdbval.cast(cast_to) else: @@ -1034,30 +1039,49 @@ def write_repr(self, out, visited): return return self._frame.write_repr(out, visited) - def print_traceback(self): - if self.is_optimized_out(): - sys.stdout.write(' %s\n' % FRAME_INFO_OPTIMIZED_OUT) - return - return self._frame.print_traceback() - class PyFramePtr: def __init__(self, gdbval): self._gdbval = gdbval + if self.is_optimized_out(): + return + self.co = self._f_code() + if self.is_shim(): + return + self.co_name = self.co.pyop_field('co_name') + self.co_filename = self.co.pyop_field('co_filename') - if not self.is_optimized_out(): + self.f_lasti = self._f_lasti() + self.co_nlocals = int_from_int(self.co.field('co_nlocals')) + pnames = self.co.field('co_localsplusnames') + self.co_localsplusnames = PyTupleObjectPtr.from_pyobject_ptr(pnames) + + @staticmethod + def get_thread_state(): + exprs = [ + '_Py_tss_gilstate', # 3.15+ + '_Py_tss_tstate', # 3.12+ (and not when GIL is released) + 'pthread_getspecific(_PyRuntime.autoTSSkey._key)', # only live programs + '((struct pthread*)$fs_base)->specific_1stblock[_PyRuntime.autoTSSkey._key].data' # x86-64 + ] + for expr in exprs: try: - self.co = self._f_code() - self.co_name = self.co.pyop_field('co_name') - self.co_filename = self.co.pyop_field('co_filename') - - self.f_lasti = self._f_lasti() - self.co_nlocals = int_from_int(self.co.field('co_nlocals')) - pnames = self.co.field('co_localsplusnames') - self.co_localsplusnames = PyTupleObjectPtr.from_pyobject_ptr(pnames) - self._is_code = True - except: - self._is_code = False + val = gdb.parse_and_eval(f'(PyThreadState*)({expr})') + except gdb.error: + continue + if int(val) != 0: + return val + return None + + @staticmethod + def get_thread_local_frame(): + thread_state = PyFramePtr.get_thread_state() + if thread_state is None: + return None + current_frame = thread_state['current_frame'] + if int(current_frame) == 0: + return None + return PyFramePtr(current_frame) def is_optimized_out(self): return self._gdbval.is_optimized_out @@ -1115,6 +1139,8 @@ def is_shim(self): return self._f_special("owner", int) == FRAME_OWNED_BY_INTERPRETER def previous(self): + if int(self._gdbval['previous']) == 0: + return None return self._f_special("previous", PyFramePtr) def iter_globals(self): @@ -1243,6 +1269,27 @@ def print_traceback(self): lineno, self.co_name.proxyval(visited))) + def print_traceback_until_shim(self, frame_index=None): + # Print traceback for _PyInterpreterFrame and return previous frame + interp_frame = self + while True: + if not interp_frame: + sys.stdout.write(' (unable to read python frame information)\n') + return None + if interp_frame.is_shim(): + return interp_frame.previous() + + if frame_index is not None: + line = interp_frame.get_truncated_repr(MAX_OUTPUT_LEN) + sys.stdout.write('#%i %s\n' % (frame_index, line)) + else: + interp_frame.print_traceback() + if not interp_frame.is_optimized_out(): + line = interp_frame.current_line() + if line is not None: + sys.stdout.write(' %s\n' % line.strip()) + interp_frame = interp_frame.previous() + def get_truncated_repr(self, maxlen): ''' Get a repr-like string for the data, but truncate it at "maxlen" bytes @@ -1855,20 +1902,10 @@ def get_selected_bytecode_frame(cls): def print_summary(self): if self.is_evalframe(): interp_frame = self.get_pyop() - while True: - if interp_frame: - if interp_frame.is_shim(): - break - line = interp_frame.get_truncated_repr(MAX_OUTPUT_LEN) - sys.stdout.write('#%i %s\n' % (self.get_index(), line)) - if not interp_frame.is_optimized_out(): - line = interp_frame.current_line() - if line is not None: - sys.stdout.write(' %s\n' % line.strip()) - else: - sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) - break - interp_frame = interp_frame.previous() + if interp_frame: + interp_frame.print_traceback_until_shim(self.get_index()) + else: + sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) else: info = self.is_other_python_frame() if info: @@ -1876,29 +1913,6 @@ def print_summary(self): else: sys.stdout.write('#%i\n' % self.get_index()) - def print_traceback(self): - if self.is_evalframe(): - interp_frame = self.get_pyop() - while True: - if interp_frame: - if interp_frame.is_shim(): - break - interp_frame.print_traceback() - if not interp_frame.is_optimized_out(): - line = interp_frame.current_line() - if line is not None: - sys.stdout.write(' %s\n' % line.strip()) - else: - sys.stdout.write(' (unable to read python frame information)\n') - break - interp_frame = interp_frame.previous() - else: - info = self.is_other_python_frame() - if info: - sys.stdout.write(' %s\n' % info) - else: - sys.stdout.write(' (not a python frame)\n') - class PyList(gdb.Command): '''List the current Python source code, if any @@ -2042,6 +2056,41 @@ def invoke(self, args, from_tty): PyUp() PyDown() + +def print_traceback_helper(full_info): + frame = Frame.get_selected_python_frame() + interp_frame = PyFramePtr.get_thread_local_frame() + if not frame and not interp_frame: + print('Unable to locate python frame') + return + + sys.stdout.write('Traceback (most recent call first):\n') + if frame: + while frame: + frame_index = frame.get_index() if full_info else None + if frame.is_evalframe(): + pyop = frame.get_pyop() + if pyop is not None: + # Use the _PyInterpreterFrame from the gdb frame + interp_frame = pyop + if interp_frame: + interp_frame = interp_frame.print_traceback_until_shim(frame_index) + else: + sys.stdout.write(' (unable to read python frame information)\n') + else: + info = frame.is_other_python_frame() + if full_info: + if info: + sys.stdout.write('#%i %s\n' % (frame_index, info)) + elif info: + sys.stdout.write(' %s\n' % info) + frame = frame.older() + else: + # Fall back to just using the thread-local frame + while interp_frame: + interp_frame = interp_frame.print_traceback_until_shim() + + class PyBacktraceFull(gdb.Command): 'Display the current python frame and all the frames within its call stack (if any)' def __init__(self): @@ -2052,15 +2101,7 @@ def __init__(self): def invoke(self, args, from_tty): - frame = Frame.get_selected_python_frame() - if not frame: - print('Unable to locate python frame') - return - - while frame: - if frame.is_python_frame(): - frame.print_summary() - frame = frame.older() + print_traceback_helper(full_info=True) PyBacktraceFull() @@ -2072,18 +2113,8 @@ def __init__(self): gdb.COMMAND_STACK, gdb.COMPLETE_NONE) - def invoke(self, args, from_tty): - frame = Frame.get_selected_python_frame() - if not frame: - print('Unable to locate python frame') - return - - sys.stdout.write('Traceback (most recent call first):\n') - while frame: - if frame.is_python_frame(): - frame.print_traceback() - frame = frame.older() + print_traceback_helper(full_info=False) PyBacktrace() From 8cf5c4d89a526e5370f1d094885021b4792d4fff Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 8 Jan 2026 09:32:20 -0500 Subject: [PATCH 6/7] gh-142908: Don't use `DK_IS_UNICODE` in interpreter (gh-142909) `DK_IS_UNICODE()` includes split keys and we don't want to specialize on those accesses.. --- Python/bytecodes.c | 8 ++++---- Python/executor_cases.c.h | 14 +++++++------- Python/generated_cases.c.h | 8 ++++---- Python/specialize.c | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 300683a9402784..5e5e818b9d3f55 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1844,7 +1844,7 @@ dummy_func( DEOPT_IF(!PyDict_CheckExact(dict)); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); } op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) @@ -1853,7 +1853,7 @@ dummy_func( DEOPT_IF(!PyDict_CheckExact(dict)); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); @@ -1873,7 +1873,7 @@ dummy_func( DEOPT_IF(!PyDict_CheckExact(dict)); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); DEOPT_IF(res_o == NULL); @@ -2668,7 +2668,7 @@ dummy_func( assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { + dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); DEOPT_IF(true); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fb8a919336bcf4..36464764e4d5ad 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7463,7 +7463,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; @@ -7488,7 +7488,7 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -7517,7 +7517,7 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); @@ -7550,7 +7550,7 @@ SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); _tos_cache2 = _stack_item_2; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; @@ -7577,7 +7577,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); @@ -7623,7 +7623,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); if (res_o == NULL) { @@ -9768,7 +9768,7 @@ assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { + dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4de75b54c9e0a1..42058066cbd12d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9213,7 +9213,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); } // _LOAD_GLOBAL_BUILTINS { @@ -9231,7 +9231,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); if (res_o == NULL) { @@ -9297,7 +9297,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - assert(DK_IS_UNICODE(keys)); + assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); @@ -10989,7 +10989,7 @@ assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { + dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); if (true) { UPDATE_MISS_STATS(STORE_ATTR); diff --git a/Python/specialize.c b/Python/specialize.c index fee54695c7f3c6..80db7d01f38f1e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1285,7 +1285,7 @@ specialize_load_global_lock_held( goto fail; } PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys; - if (!DK_IS_UNICODE(globals_keys)) { + if (globals_keys->dk_kind != DICT_KEYS_UNICODE) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); goto fail; } @@ -1320,7 +1320,7 @@ specialize_load_global_lock_held( goto fail; } PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys; - if (!DK_IS_UNICODE(builtin_keys)) { + if (builtin_keys->dk_kind != DICT_KEYS_UNICODE) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); goto fail; } From cea2d2475d3eec9f4fd350ef9eb2ba43da1943a5 Mon Sep 17 00:00:00 2001 From: Heikki Toivonen <308110+heikkitoivonen@users.noreply.github.com> Date: Thu, 8 Jan 2026 07:28:02 -0800 Subject: [PATCH 7/7] gh-143445: Optimize deepcopy for 1.04x speedup (#143449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gains according to pyperformance: ``` deepcopy: Mean +- std dev: 411 us +- 2 us -> 396 us +- 3 us: 1.04x faster Significant (t=28.94) deepcopy_reduce: Mean +- std dev: 4.38 us +- 0.05 us -> 4.23 us +- 0.04 us: 1.04x faster Significant (t=20.05) ``` Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/copy.py | 2 +- Misc/ACKS | 1 + .../next/Library/2026-01-05-12-20-42.gh-issue-143445.rgxnbL.rst | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-01-05-12-20-42.gh-issue-143445.rgxnbL.rst diff --git a/Lib/copy.py b/Lib/copy.py index fff7e93c2a1b89..4c024ab5311d2d 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -230,7 +230,7 @@ def _reconstruct(x, memo, func, args, *, deepcopy=deepcopy): deep = memo is not None if deep and args: - args = (deepcopy(arg, memo) for arg in args) + args = [deepcopy(arg, memo) for arg in args] y = func(*args) if deep: memo[id(x)] = y diff --git a/Misc/ACKS b/Misc/ACKS index 671fcf88c75af9..63ddfb89071c0b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1931,6 +1931,7 @@ James Tocknell Bennett Todd R Lindsay Todd Eugene Toder +Heikki Toivonen Erik Tollerud Stephen Tonkin Matias Torchinsky diff --git a/Misc/NEWS.d/next/Library/2026-01-05-12-20-42.gh-issue-143445.rgxnbL.rst b/Misc/NEWS.d/next/Library/2026-01-05-12-20-42.gh-issue-143445.rgxnbL.rst new file mode 100644 index 00000000000000..f5dea2e49afe2a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-05-12-20-42.gh-issue-143445.rgxnbL.rst @@ -0,0 +1 @@ +Speed up :func:`copy.deepcopy` by 1.04x.