From 4d02f31cdd45d81b95540d9076222b709d4f2335 Mon Sep 17 00:00:00 2001 From: Timon Viola <44016238+timonviola@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:27:13 +0200 Subject: [PATCH 1/9] gh-118350: Fix support of elements "textarea" and "title" in HTMLParser (#135310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Serhiy Storchaka Co-authored-by: Łukasz Langa --- Lib/html/parser.py | 20 +++- Lib/test/test_htmlparser.py | 96 +++++++++++++++++++ ...-06-09-20-38-25.gh-issue-118350.KgWCcP.rst | 2 + 3 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 7eea885cfe63c5..5d03c98df5cdd0 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -128,6 +128,7 @@ class HTMLParser(_markupbase.ParserBase): """ CDATA_CONTENT_ELEMENTS = ("script", "style") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") def __init__(self, *, convert_charrefs=True): """Initialize and reset this instance. @@ -145,6 +146,7 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True super().reset() def feed(self, data): @@ -166,14 +168,20 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'])' % self.cdata_elem, - re.IGNORECASE|re.ASCII) + self._escapable = escapable + if escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -206,7 +214,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -308,7 +316,7 @@ def goahead(self, end): assert 0, "interesting.search() lied" # end while if end and i < n: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:n])) else: self.handle_data(rawdata[i:n]) @@ -420,6 +428,8 @@ def parse_starttag(self, i): self.handle_starttag(tag, attrs) if tag in self.CDATA_CONTENT_ELEMENTS: self.set_cdata_mode(tag) + elif tag in self.RCDATA_CONTENT_ELEMENTS: + self.set_cdata_mode(tag, escapable=True) return endpos # Internal -- check to see if we have a complete starttag; return end diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 47c0752fb517b9..380bbe40177ec5 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -317,6 +317,49 @@ def test_style_content(self, content): ("data", content), ("endtag", "style")]) + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /title>', + '', + '', + '', + '', + '', + ]) + def test_title_content(self, content): + source = f"{content}" + self._run_check(source, [ + ("starttag", "title", []), + ("data", content), + ("endtag", "title"), + ]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /textarea>', + '', + '', + '', + '', + ]) + def test_textarea_content(self, content): + source = f"" + self._run_check(source, [ + ("starttag", "textarea", []), + ("data", content), + ("endtag", "textarea"), + ]) + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', 'script/', 'script foo=bar', 'script foo=">"']) def test_script_closing_tag(self, endtag): @@ -346,6 +389,38 @@ def test_style_closing_tag(self, endtag): ("endtag", "style")], collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('endtag', ['title', 'TITLE', 'title ', 'title\n', + 'title/', 'title foo=bar', 'title foo=">"']) + def test_title_closing_tag(self, endtag): + content = "Egg & Spam" + s = f'{content}</{endtag}>' + self._run_check(s, [("starttag", "title", []), + ('data', '<!-- not a comment --><i>Egg & Spam</i>'), + ("endtag", "title")], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', '<!-- not a comment --><i>Egg '), + ('entityref', 'amp'), + ('data', ' Spam</i>'), + ("endtag", "title")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('endtag', ['textarea', 'TEXTAREA', 'textarea ', 'textarea\n', + 'textarea/', 'textarea foo=bar', 'textarea foo=">"']) + def test_textarea_closing_tag(self, endtag): + content = "<!-- not a comment --><i>Egg & Spam</i>" + s = f'<TexTarEa>{content}</{endtag}>' + self._run_check(s, [("starttag", "textarea", []), + ('data', '<!-- not a comment --><i>Egg & Spam</i>'), + ("endtag", "textarea")], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "textarea", []), + ('data', '<!-- not a comment --><i>Egg '), + ('entityref', 'amp'), + ('data', ' Spam</i>'), + ("endtag", "textarea")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('tail,end', [ ('', False), ('<', False), @@ -363,6 +438,27 @@ def test_eof_in_script(self, tail, end): ("data", content if end else content + tail)], collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</t', False), + ('</title', False), + ('" '' diff --git a/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst new file mode 100644 index 00000000000000..6ad3caf33b2201 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst @@ -0,0 +1,2 @@ +Fix support of escapable raw text mode (elements "textarea" and "title") +in :class:`html.parser.HTMLParser`. From aafbdb5df5439adc1106ced068cf87683ae68b9e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 22 Jul 2025 13:53:05 +0200 Subject: [PATCH 2/9] gh-133742: Fix test_sysconfig and test_build_details for relocated directories (#136987) --- Lib/test/test_build_details.py | 9 +++++++++ Lib/test/test_sysconfig.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index ba4b8c5aa9b58e..691fd0bb98c097 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -124,6 +124,10 @@ def test_location(self): def test_base_interpreter(self): value = self.key('base_interpreter') + # Skip check if installation is relocated + if sysconfig._installation_is_relocated(): + self.skipTest("Installation is relocated") + self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) @needs_installed_python @@ -133,6 +137,11 @@ def test_base_interpreter(self): ) def test_c_api(self): value = self.key('c_api') + + # Skip check if installation is relocated + if sysconfig._installation_is_relocated(): + self.skipTest("Installation is relocated") + self.assertTrue(os.path.exists(os.path.join(value['headers'], 'Python.h'))) version = sysconfig.get_config_var('VERSION') self.assertTrue(os.path.exists(os.path.join(value['pkgconfig_path'], f'python-{version}.pc'))) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2eb8de4b29fe96..4aaef5b142931e 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -708,7 +708,7 @@ def test_sysconfigdata_json(self): ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) if sysconfig._installation_is_relocated(): - ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase', 'srcdir'} for key in ignore_keys: json_config_vars.pop(key, None) From 12d2f373b9f70ce43a985ada2832ca31ca66fc20 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 22 Jul 2025 15:05:26 +0200 Subject: [PATCH 3/9] gh-124621: Emscripten: Fix __syscall_ioctl patch (GH-136993) If there is an error, we have to return `-errno` not positive errno. Included in backport of GH-136931: #136988 --- Python/emscripten_syscalls.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index 404d98d492a655..98ee44276e53e0 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -1,6 +1,10 @@ #include "emscripten.h" #include "stdio.h" +// All system calls: return nonnegative number on success, return -errno on +// failure. Negative results get stored back into errno here: +// https://github.com/emscripten-core/emscripten/blob/main/system/lib/libc/musl/src/internal/syscall_ret.c#L7 + // If we're running in node, report the UID of the user in the native system as // the UID of the user. Since the nodefs will report the uid correctly, if we // don't make getuid report it correctly too we'll see some permission errors. @@ -302,7 +306,7 @@ int __syscall_ioctl(int fd, int request, void* varargs) { int flags = fcntl(fd, F_GETFL, 0); int nonblock = **((int**)varargs); if (flags < 0) { - return errno; + return -errno; } if (nonblock) { flags |= O_NONBLOCK; @@ -311,7 +315,7 @@ int __syscall_ioctl(int fd, int request, void* varargs) { } int res = fcntl(fd, F_SETFL, flags); if (res < 0) { - return errno; + return -errno; } return res; } From b31e5d6de19403cabe7e33aa8866e05c895db24a Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 22 Jul 2025 16:38:33 +0300 Subject: [PATCH 4/9] gh-136437: Convert more ``os.path`` functions to positional-only in the docs (#136970) --- Doc/library/os.path.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index d59f91f2aa8000..d27023cde4cfe9 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -94,7 +94,7 @@ the :mod:`glob` module.) Any iterable can now be passed, rather than just sequences. -.. function:: commonprefix(list) +.. function:: commonprefix(list, /) Return the longest path prefix (taken character-by-character) that is a prefix of all paths in *list*. If *list* is empty, return the empty string @@ -199,14 +199,14 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getatime(path) +.. function:: getatime(path, /) Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. -.. function:: getmtime(path) +.. function:: getmtime(path, /) Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). @@ -216,7 +216,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getctime(path) +.. function:: getctime(path, /) Return the system's ctime which, on some systems (like Unix) is the time of the last metadata change, and, on others (like Windows), is the creation time for *path*. @@ -228,7 +228,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getsize(path) +.. function:: getsize(path, /) Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -351,7 +351,7 @@ the :mod:`glob` module.) .. versionadded:: 3.13 -.. function:: join(path, *paths) +.. function:: join(path, /, *paths) Join one or more path segments intelligently. The return value is the concatenation of *path* and all members of *\*paths*, with exactly one @@ -402,7 +402,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: realpath(path, *, strict=False) +.. function:: realpath(path, /, *, strict=False) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating @@ -471,7 +471,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samefile(path1, path2) +.. function:: samefile(path1, path2, /) Return ``True`` if both pathname arguments refer to the same file or directory. This is determined by the device number and i-node number and raises an @@ -498,7 +498,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samestat(stat1, stat2) +.. function:: samestat(stat1, stat2, /) Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. These structures may have been returned by :func:`os.fstat`, From 3a89dfe32bec016de480c2faed5effc2bbf48a45 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 22 Jul 2025 16:39:50 +0300 Subject: [PATCH 5/9] Revert "gh-112068: C API: Add support of nullable arguments in PyArg_Parse (GH-121303)" (#136991) --- Doc/c-api/arg.rst | 15 -- Doc/whatsnew/3.14.rst | 5 - Lib/test/test_capi/test_getargs.py | 117 --------- Lib/test/test_mmap.py | 2 +- Misc/NEWS.d/3.14.0b1.rst | 2 +- ...-07-22-15-18-08.gh-issue-112068.4WvT-8.rst | 1 + Modules/_ctypes/_ctypes.c | 12 +- Modules/_interpretersmodule.c | 9 +- Modules/_json.c | 13 +- Modules/_threadmodule.c | 18 +- Modules/mmapmodule.c | 15 +- Python/getargs.c | 247 ++++++------------ 12 files changed, 139 insertions(+), 317 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 803572afc39ae3..3429a4eb652709 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C: ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s?`` with the C pointer was initialized to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. - It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer` - structure was initialized to ``NULL``. ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s#?`` with the C pointer was initialized to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a @@ -394,17 +390,6 @@ Other objects Non-tuple sequences are deprecated if *items* contains format units which store a borrowed buffer or a borrowed reference. -``unit?`` (anything or ``None``) [*matching-variable(s)*] - ``?`` modifies the behavior of the preceding format unit. - The C variable(s) corresponding to that parameter should be initialized - to their default value --- when the argument is ``None``, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding - C variable(s). - If the argument is not ``None``, it is parsed according to the specified - format unit. - - .. versionadded:: 3.14 - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 6bdf4aee709ad3..e45a2bfa485c50 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -2940,11 +2940,6 @@ New features file. (Contributed by Victor Stinner in :gh:`127350`.) -* Add support of nullable arguments in :c:func:`PyArg_ParseTuple` and - similar functions. - Adding ``?`` after any format unit makes ``None`` be accepted as a value. - (Contributed by Serhiy Storchaka in :gh:`112068`.) - * The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and similar functions now use :meth:`~object.__index__` if available, like all other integer formats. diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index dc2000089684a6..0b2473bac2be11 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1429,123 +1429,6 @@ def test_nested_sequence(self): "argument 1 must be sequence of length 1, not 0"): parse(([],), {}, '(' + f + ')', ['a']) - def test_specific_type_errors(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, expected, got='list'): - errmsg = f'must be {expected}, not {got}' - with self.assertRaisesRegex(TypeError, errmsg): - parse((arg,), {}, format, ['a']) - - check('k', [], 'int') - check('k?', [], 'int or None') - check('K', [], 'int') - check('K?', [], 'int or None') - check('c', [], 'a byte string of length 1') - check('c?', [], 'a byte string of length 1 or None') - check('c', b'abc', 'a byte string of length 1', - 'a bytes object of length 3') - check('c?', b'abc', 'a byte string of length 1 or None', - 'a bytes object of length 3') - check('c', bytearray(b'abc'), 'a byte string of length 1', - 'a bytearray object of length 3') - check('c?', bytearray(b'abc'), 'a byte string of length 1 or None', - 'a bytearray object of length 3') - check('C', [], 'a unicode character') - check('C?', [], 'a unicode character or None') - check('C', 'abc', 'a unicode character', - 'a string of length 3') - check('C?', 'abc', 'a unicode character or None', - 'a string of length 3') - check('s', [], 'str') - check('s?', [], 'str or None') - check('z', [], 'str or None') - check('z?', [], 'str or None') - check('es', [], 'str') - check('es?', [], 'str or None') - check('es#', [], 'str') - check('es#?', [], 'str or None') - check('et', [], 'str, bytes or bytearray') - check('et?', [], 'str, bytes, bytearray or None') - check('et#', [], 'str, bytes or bytearray') - check('et#?', [], 'str, bytes, bytearray or None') - check('w*', [], 'read-write bytes-like object') - check('w*?', [], 'read-write bytes-like object or None') - check('S', [], 'bytes') - check('S?', [], 'bytes or None') - check('U', [], 'str') - check('U?', [], 'str or None') - check('Y', [], 'bytearray') - check('Y?', [], 'bytearray or None') - check('(OO)', 42, '2-item tuple', 'int') - check('(OO)?', 42, '2-item tuple or None', 'int') - check('(OO)', (1, 2, 3), 'tuple of length 2', '3') - - def test_nullable(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, allows_none=False): - # Because some format units (such as y*) require cleanup, - # we force the parsing code to perform the cleanup by adding - # an argument that always fails. - # By checking for an exception, we ensure that the parsing - # of the first argument was successful. - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (None, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + 'b', ['a', 'b']) - self.assertRaises(OverflowError if allows_none else TypeError, parse, - (None, 256), {}, format + 'b', ['a', 'b']) - - check('b', 42) - check('B', 42) - check('h', 42) - check('H', 42) - check('i', 42) - check('I', 42) - check('n', 42) - check('l', 42) - check('k', 42) - check('L', 42) - check('K', 42) - check('f', 2.5) - check('d', 2.5) - check('D', 2.5j) - check('c', b'a') - check('C', 'a') - check('p', True, allows_none=True) - check('y', b'buffer') - check('y*', b'buffer') - check('y#', b'buffer') - check('s', 'string') - check('s*', 'string') - check('s#', 'string') - check('z', 'string', allows_none=True) - check('z*', 'string', allows_none=True) - check('z#', 'string', allows_none=True) - check('w*', bytearray(b'buffer')) - check('U', 'string') - check('S', b'bytes') - check('Y', bytearray(b'bytearray')) - check('O', object, allows_none=True) - - check('(OO)', (1, 2)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)?O)', ['a']), (1, 2, 3)) - self.assertEqual(parse(((None, 3),), {}, '((OO)?O)', ['a']), (NULL, NULL, 3)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)O)', ['a']), (1, 2, 3)) - self.assertRaises(TypeError, parse, ((None, 3),), {}, '((OO)O)', ['a']) - - parse((None,), {}, 'es?', ['a']) - parse((None,), {}, 'es#?', ['a']) - parse((None,), {}, 'et?', ['a']) - parse((None,), {}, 'et#?', ['a']) - parse((None,), {}, 'O!?', ['a']) - parse((None,), {}, 'O&?', ['a']) - - # TODO: More tests for es?, es#?, et?, et#?, O!, O& - @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') def test_gh_119213(self): rc, out, err = script_helper.assert_python_ok("-c", """if True: diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index fd4197b7086976..b2a299ed172967 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -732,7 +732,7 @@ def test_tagname(self): m2.close() m1.close() - with self.assertRaisesRegex(TypeError, 'must be str or None'): + with self.assertRaisesRegex(TypeError, 'tagname'): mmap.mmap(-1, 8, tagname=1) @cpython_only diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index 02ceb82b556386..5d03d429f9ee14 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -2012,7 +2012,7 @@ interpreter. .. nonce: ofI5Fl .. section: C API -Add support of nullable arguments in :c:func:`PyArg_Parse` and similar +[Reverted in :gh:`136991`] Add support of nullable arguments in :c:func:`PyArg_Parse` and similar functions. Adding ``?`` after any format unit makes ``None`` be accepted as a value. diff --git a/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst new file mode 100644 index 00000000000000..018c5c7880c001 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst @@ -0,0 +1 @@ +Revert support of nullable arguments in :c:func:`PyArg_Parse`. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 7e8a133caa72ac..4bd3e380b3bc4b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3937,7 +3937,9 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) PyObject *name = Py_None; PyObject *defval; PyObject *typ; - if (!PyArg_ParseTuple(item, "i|U?O", &flag, &name, &defval)) { + if (!PyArg_ParseTuple(item, "i|OO", &flag, &name, &defval) || + !(name == Py_None || PyUnicode_Check(name))) + { PyErr_SetString(PyExc_TypeError, "paramflags must be a sequence of (int [,string [,value]]) tuples"); return 0; @@ -4002,8 +4004,10 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) void *handle; PyObject *paramflags = NULL; - if (!PyArg_ParseTuple(args, "O|O?", &ftuple, ¶mflags)) + if (!PyArg_ParseTuple(args, "O|O", &ftuple, ¶mflags)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ftuple = PySequence_Tuple(ftuple); if (!ftuple) @@ -4135,8 +4139,10 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) GUID *iid = NULL; Py_ssize_t iid_len = 0; - if (!PyArg_ParseTuple(args, "is|O?z#", &index, &name, ¶mflags, &iid, &iid_len)) + if (!PyArg_ParseTuple(args, "is|Oz#", &index, &name, ¶mflags, &iid, &iid_len)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 9426ce72733c28..faf3b25b68c4eb 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -1415,11 +1415,14 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) PyObject *idobj = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O?|$p:get_config", kwlist, + "O|$p:get_config", kwlist, &idobj, &restricted)) { return NULL; } + if (idobj == Py_None) { + idobj = NULL; + } int reqready = 0; PyInterpreterState *interp = \ @@ -1536,14 +1539,14 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) static char *kwlist[] = {"exc", NULL}; PyObject *exc_arg = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O?:capture_exception", kwlist, + "|O:capture_exception", kwlist, &exc_arg)) { return NULL; } PyObject *exc = exc_arg; - if (exc == NULL) { + if (exc == NULL || exc == Py_None) { exc = PyErr_GetRaisedException(); if (exc == NULL) { Py_RETURN_NONE; diff --git a/Modules/_json.c b/Modules/_json.c index 6b5f6ea42df4d1..7580b589e2d937 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1222,16 +1222,23 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; - PyObject *markers = Py_None, *defaultfn, *encoder, *indent, *key_separator; + PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; PyObject *item_separator; int sort_keys, skipkeys, allow_nan; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!?OOOUUppp:make_encoder", kwlist, - &PyDict_Type, &markers, &defaultfn, &encoder, &indent, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppp:make_encoder", kwlist, + &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) return NULL; + if (markers != Py_None && !PyDict_Check(markers)) { + PyErr_Format(PyExc_TypeError, + "make_encoder() argument 1 must be dict or None, " + "not %.200s", Py_TYPE(markers)->tp_name); + return NULL; + } + s = (PyEncoderObject *)type->tp_alloc(type, 0); if (s == NULL) return NULL; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 3540fead8e8e74..f82ad6870f850f 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -681,12 +681,12 @@ PyThreadHandleObject_join(PyObject *op, PyObject *args) PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); PyObject *timeout_obj = NULL; - if (!PyArg_ParseTuple(args, "|O?:join", &timeout_obj)) { + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; } PyTime_t timeout_ns = -1; - if (timeout_obj != NULL) { + if (timeout_obj != NULL && timeout_obj != Py_None) { if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { return NULL; @@ -1957,10 +1957,10 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, PyObject *func = NULL; int daemon = 1; thread_module_state *state = get_thread_state(module); - PyObject *hobj = Py_None; + PyObject *hobj = NULL; if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, - "O|O!?p:start_joinable_thread", keywords, - &func, state->thread_handle_type, &hobj, &daemon)) { + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { return NULL; } @@ -1970,6 +1970,14 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, return NULL; } + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, hobj) < 0) { return NULL; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 142ff1a21316ab..0cb4b62d734550 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -23,6 +23,7 @@ #endif #include +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -529,7 +530,7 @@ mmap_read_method(PyObject *op, PyObject *args) mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|n?:read", &num_bytes)) + if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) return NULL; CHECK_VALID(NULL); @@ -1723,7 +1724,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - PyObject *tagname = NULL; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; @@ -1733,7 +1734,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) "tagname", "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|U?iL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; @@ -1866,7 +1867,13 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL) { + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { Py_DECREF(m_obj); diff --git a/Python/getargs.c b/Python/getargs.c index 02f5c0e37ee42a..c119ca5c35398b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,8 +1,6 @@ /* New getargs implementation */ -#include - #define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() @@ -468,12 +466,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; int i; Py_ssize_t len; - bool nullable = false; int istuple = PyTuple_Check(arg); int mustbetuple = istuple; - assert(*format == '('); - format++; for (;;) { int c = *format++; if (c == '(') { @@ -482,12 +477,8 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, level++; } else if (c == ')') { - if (level == 0) { - if (*format == '?') { - nullable = true; - } + if (level == 0) break; - } level--; } else if (c == ':' || c == ';' || c == '\0') @@ -524,13 +515,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - if (arg == Py_None && nullable) { - const char *msg = skipitem(p_format, p_va, flags); - if (msg != NULL) { - levels[0] = 0; - } - return msg; - } if (istuple) { /* fallthrough */ } @@ -539,10 +523,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be %d-item tuple%s, not %.50s", - n, - nullable ? " or None" : "", - arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); + "must be %d-item tuple, not %.50s", + n, + arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); return msgbuf; } else { @@ -579,7 +562,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return msgbuf; } - format = *p_format + 1; + format = *p_format; for (i = 0; i < n; i++) { const char *msg; PyObject *item = PyTuple_GET_ITEM(arg, i); @@ -594,10 +577,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - format++; - if (*format == '?') { - format++; - } *p_format = format; if (!istuple) { Py_DECREF(arg); @@ -616,8 +595,11 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; if (*format == '(' /* ')' */) { + format++; msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, bufsize, freelist); + if (msg == NULL) + format++; } else { msg = convertsimple(arg, &format, p_va, flags, @@ -647,7 +629,7 @@ _PyArg_BadArgument(const char *fname, const char *displayname, } static const char * -converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) +converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); assert(arg != NULL); @@ -657,23 +639,20 @@ converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, siz } else { PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s", expected, - nullable ? " or None" : "", + "must be %.50s, not %.50s", expected, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); } return msgbuf; } static const char * -convertcharerr(bool nullable, const char *expected, const char *what, Py_ssize_t size, +convertcharerr(const char *expected, const char *what, Py_ssize_t size, char *msgbuf, size_t bufsize) { assert(expected != NULL); PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s of length %zd", - expected, - nullable ? " or None" : "", - what, size); + "must be %.50s, not %.50s of length %zd", + expected, what, size); return msgbuf; } @@ -693,26 +672,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { #define RETURN_ERR_OCCURRED return msgbuf -#define HANDLE_NULLABLE \ - if (*format == '?') { \ - format++; \ - if (arg == Py_None) { \ - break; \ - } \ - nullable = true; \ - } - const char *format = *p_format; char c = *format++; const char *sarg; - bool nullable = false; switch (c) { case 'b': { /* unsigned byte -- very short int */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -726,6 +694,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, "unsigned byte integer is greater than maximum"); RETURN_ERR_OCCURRED; } + else *p = (unsigned char) ival; break; } @@ -733,7 +702,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned char), Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_ALLOW_INDEX | @@ -753,7 +721,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'h': {/* signed short int */ short *p = va_arg(*p_va, short *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -775,7 +742,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'H': { /* short int sized bitfield, both signed and unsigned allowed */ unsigned short *p = va_arg(*p_va, unsigned short *); - HANDLE_NULLABLE; Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned short), Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_ALLOW_INDEX | @@ -795,7 +761,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'i': {/* signed int */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -817,7 +782,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'I': { /* int sized bitfield, both signed and unsigned allowed */ unsigned int *p = va_arg(*p_va, unsigned int *); - HANDLE_NULLABLE; Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned int), Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_ALLOW_INDEX | @@ -839,7 +803,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { PyObject *iobj; Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); - HANDLE_NULLABLE; Py_ssize_t ival = -1; iobj = _PyNumber_Index(arg); if (iobj != NULL) { @@ -853,7 +816,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'l': {/* long int */ long *p = va_arg(*p_va, long *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -864,9 +826,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); - HANDLE_NULLABLE; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned long), Py_ASNATIVEBYTES_NATIVE_ENDIAN | @@ -887,7 +848,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'L': {/* long long */ long long *p = va_arg( *p_va, long long * ); - HANDLE_NULLABLE; long long ival = PyLong_AsLongLong(arg); if (ival == (long long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -898,9 +858,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'K': { /* long long sized bitfield */ unsigned long long *p = va_arg(*p_va, unsigned long long *); - HANDLE_NULLABLE; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned long long), Py_ASNATIVEBYTES_NATIVE_ENDIAN | @@ -921,7 +880,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'f': {/* float */ float *p = va_arg(*p_va, float *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -932,7 +890,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'd': {/* double */ double *p = va_arg(*p_va, double *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -943,7 +900,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'D': {/* complex double */ Py_complex *p = va_arg(*p_va, Py_complex *); - HANDLE_NULLABLE; Py_complex cval; cval = PyComplex_AsCComplex(arg); if (PyErr_Occurred()) @@ -955,10 +911,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) { if (PyBytes_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytes object", PyBytes_GET_SIZE(arg), msgbuf, bufsize); } @@ -966,28 +921,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (PyByteArray_Check(arg)) { if (PyByteArray_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytearray object", PyByteArray_GET_SIZE(arg), msgbuf, bufsize); } *p = PyByteArray_AS_STRING(arg)[0]; } else - return converterr(nullable, "a byte string of length 1", arg, msgbuf, bufsize); + return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; } case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int kind; const void *data; if (!PyUnicode_Check(arg)) - return converterr(nullable, "a unicode character", arg, msgbuf, bufsize); + return converterr("a unicode character", arg, msgbuf, bufsize); if (PyUnicode_GET_LENGTH(arg) != 1) { - return convertcharerr(nullable, "a unicode character", + return convertcharerr("a unicode character", "a string", PyUnicode_GET_LENGTH(arg), msgbuf, bufsize); } @@ -1000,7 +954,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'p': {/* boolean *p*redicate */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int val = PyObject_IsTrue(arg); if (val > 0) *p = 1; @@ -1019,31 +972,24 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count; if (*format == '*') { - format++; - HANDLE_NULLABLE; if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); + format++; if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } - else if (*format == '#') { + count = convertbuffer(arg, (const void **)p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + if (*format == '#') { Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); *psize = count; - } - else { - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + format++; + } else { if (strlen(*p) != (size_t)count) { PyErr_SetString(PyExc_ValueError, "embedded null byte"); RETURN_ERR_OCCURRED; @@ -1059,35 +1005,32 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* "s*" or "z*" */ Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); PyBuffer_FillInfo(p, arg, (void *)sarg, len, 1, 0); } else { /* any bytes-like object */ const char *buf; if (getbuffer(arg, p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); } if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } + format++; } else if (*format == '#') { /* a string or read-only bytes-like object */ /* "s#" or "z#" */ const void **p = (const void **)va_arg(*p_va, const char **); Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) { *p = NULL; *psize = 0; @@ -1096,7 +1039,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); *p = sarg; *psize = len; @@ -1106,22 +1049,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); *psize = count; } + format++; } else { /* "s" or "z" */ const char **p = va_arg(*p_va, const char **); Py_ssize_t len; sarg = NULL; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); if (strlen(sarg) != (size_t)len) { PyErr_SetString(PyExc_ValueError, "embedded null character"); @@ -1130,7 +1073,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p = sarg; } else - return converterr(c == 'z' || nullable, "str", + return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); } break; @@ -1159,46 +1102,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, recode_strings = 0; else return converterr( - nullable, "(unknown parser marker combination)", + "(unknown parser marker combination)", arg, msgbuf, bufsize); buffer = (char **)va_arg(*p_va, char **); format++; if (buffer == NULL) - return converterr(nullable, "(buffer is NULL)", + return converterr("(buffer is NULL)", arg, msgbuf, bufsize); - Py_ssize_t *psize = NULL; - if (*format == '#') { - /* Using buffer length parameter '#': - - - if *buffer is NULL, a new buffer of the - needed size is allocated and the data - copied into it; *buffer is updated to point - to the new buffer; the caller is - responsible for PyMem_Free()ing it after - usage - - - if *buffer is not NULL, the data is - copied to *buffer; *buffer_len has to be - set to the size of the buffer on input; - buffer overflow is signalled with an error; - buffer has to provide enough room for the - encoded string plus the trailing 0-byte - - - in both cases, *buffer_len is updated to - the size of the buffer /excluding/ the - trailing 0-byte - - */ - psize = va_arg(*p_va, Py_ssize_t*); - - format++; - if (psize == NULL) { - return converterr( - nullable, "(buffer_len is NULL)", - arg, msgbuf, bufsize); - } - } - HANDLE_NULLABLE; /* Encode object */ if (!recode_strings && @@ -1219,7 +1129,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, encoding, NULL); if (s == NULL) - return converterr(nullable, "(encoding failed)", + return converterr("(encoding failed)", arg, msgbuf, bufsize); assert(PyBytes_Check(s)); size = PyBytes_GET_SIZE(s); @@ -1229,15 +1139,42 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { return converterr( - nullable, - recode_strings ? "str" - : nullable ? "str, bytes, bytearray" - : "str, bytes or bytearray", + recode_strings ? "str" : "str, bytes or bytearray", arg, msgbuf, bufsize); } /* Write output; output is guaranteed to be 0-terminated */ - if (psize != NULL) { + if (*format == '#') { + /* Using buffer length parameter '#': + + - if *buffer is NULL, a new buffer of the + needed size is allocated and the data + copied into it; *buffer is updated to point + to the new buffer; the caller is + responsible for PyMem_Free()ing it after + usage + + - if *buffer is not NULL, the data is + copied to *buffer; *buffer_len has to be + set to the size of the buffer on input; + buffer overflow is signalled with an error; + buffer has to provide enough room for the + encoded string plus the trailing 0-byte + + - in both cases, *buffer_len is updated to + the size of the buffer /excluding/ the + trailing 0-byte + + */ + Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); + + format++; + if (psize == NULL) { + Py_DECREF(s); + return converterr( + "(buffer_len is NULL)", + arg, msgbuf, bufsize); + } if (*buffer == NULL) { *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { @@ -1248,7 +1185,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } } else { @@ -1282,7 +1219,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if ((Py_ssize_t)strlen(ptr) != size) { Py_DECREF(s); return converterr( - nullable, "encoded string without null bytes", + "encoded string without null bytes", arg, msgbuf, bufsize); } *buffer = PyMem_NEW(char, size + 1); @@ -1293,7 +1230,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } memcpy(*buffer, ptr, size+1); @@ -1304,32 +1241,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'S': { /* PyBytes object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) *p = arg; else - return converterr(nullable, "bytes", arg, msgbuf, bufsize); + return converterr("bytes", arg, msgbuf, bufsize); break; } case 'Y': { /* PyByteArray object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyByteArray_Check(arg)) *p = arg; else - return converterr(nullable, "bytearray", arg, msgbuf, bufsize); + return converterr("bytearray", arg, msgbuf, bufsize); break; } case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyUnicode_Check(arg)) { *p = arg; } else - return converterr(nullable, "str", arg, msgbuf, bufsize); + return converterr("str", arg, msgbuf, bufsize); break; } @@ -1340,11 +1274,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, type = va_arg(*p_va, PyTypeObject*); p = va_arg(*p_va, PyObject **); format++; - HANDLE_NULLABLE; if (PyType_IsSubtype(Py_TYPE(arg), type)) *p = arg; else - return converterr(nullable, type->tp_name, arg, msgbuf, bufsize); + return converterr(type->tp_name, arg, msgbuf, bufsize); } else if (*format == '&') { @@ -1353,18 +1286,16 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, void *addr = va_arg(*p_va, void *); int res; format++; - HANDLE_NULLABLE; if (! (res = (*convert)(arg, addr))) - return converterr(nullable, "(unspecified)", + return converterr("(unspecified)", arg, msgbuf, bufsize); if (res == Py_CLEANUP_SUPPORTED && addcleanup(addr, freelist, convert) == -1) - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } else { p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; *p = arg; } break; @@ -1376,30 +1307,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format != '*') return converterr( - nullable, "(invalid use of 'w' format character)", + "(invalid use of 'w' format character)", arg, msgbuf, bufsize); format++; - HANDLE_NULLABLE; /* Caller is interested in Py_buffer, and the object supports it directly. The request implicitly asks for PyBUF_SIMPLE, so the result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - return converterr(nullable, "read-write bytes-like object", + return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } default: - return converterr(nullable, "(impossible)", arg, msgbuf, bufsize); + return converterr("(impossible)", arg, msgbuf, bufsize); } @@ -2794,9 +2724,6 @@ skipitem(const char **p_format, va_list *p_va, int flags) return "impossible"; } - if (*format == '?') { - format++; - } *p_format = format; return NULL; From c13cc4af793ba4ae27521df1693653920cafbf99 Mon Sep 17 00:00:00 2001 From: Victorien <65306057+Viicos@users.noreply.github.com> Date: Tue, 22 Jul 2025 15:48:02 +0200 Subject: [PATCH 6/9] Fix code example in `annotationlib` documentation (#136972) --- Doc/library/annotationlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 7dfc11449a6cbc..981d89be7d58d6 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -511,7 +511,7 @@ code execution even with no access to any globals or builtins. For example: >>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass ... - >>> annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + >>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING) Hello world {'x': 'None'} From b6d324224474c54061a6aaeace50bc5666dc1779 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 22 Jul 2025 15:48:58 +0200 Subject: [PATCH 7/9] GH-136975: Emend a spelling error (algorthm -> algorithm) (#136999) --- Modules/_zstd/_zstdmodule.c | 2 +- Modules/_zstd/_zstdmodule.h | 2 +- Modules/_zstd/buffer.h | 2 +- Modules/_zstd/compressor.c | 2 +- Modules/_zstd/decompressor.c | 2 +- Modules/_zstd/zstddict.c | 2 +- Modules/_zstd/zstddict.h | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index d75c0779474a82..8af6156a0da575 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 4e8f708f2232c7..82226ff8718e6b 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* Declarations shared between different parts of the _zstd module*/ diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 4c885fa0d720fd..0ac7bcb4ddc416 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef ZSTD_BUFFER_H #define ZSTD_BUFFER_H diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index bc9e6eff89af68..508b136817872b 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdCompressor class definitions */ diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index c53d6e4cb05cf0..b00ee05d2f51bf 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDecompressor class definition */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 14f74aaed46ec5..35d6ca8e55a265 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDict class definitions */ diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index 4a403416dbd4a3..e0d3f46b2b14a6 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef ZSTD_DICT_H #define ZSTD_DICT_H From a6678005587526dac969b4cadd016c7da14f018b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Naz=C4=B1m=20Can=20Alt=C4=B1nova?= Date: Tue, 22 Jul 2025 17:47:24 +0200 Subject: [PATCH 8/9] gh-136459: Add perf trampoline support for macOS (#136461) --- Doc/c-api/perfmaps.rst | 9 +- Doc/howto/perf_profiling.rst | 56 ++-- Lib/test/test_perfmaps.py | 12 +- Lib/test/test_samply_profiler.py | 244 ++++++++++++++++++ Misc/ACKS | 1 + ...-07-09-11-15-42.gh-issue-136459.m4Udh8.rst | 3 + Python/asm_trampoline.S | 13 + Python/perf_jit_trampoline.c | 35 ++- configure | 2 + configure.ac | 3 +- 10 files changed, 351 insertions(+), 27 deletions(-) create mode 100644 Lib/test/test_samply_profiler.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 77b5e3c0876bbb..76a1e9f528dc70 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -5,11 +5,12 @@ Support for Perf Maps ---------------------- -On supported platforms (as of this writing, only Linux), the runtime can take +On supported platforms (Linux and macOS), the runtime can take advantage of *perf map files* to make Python functions visible to an external -profiling tool (such as `perf `_). -A running process may create a file in the ``/tmp`` directory, which contains entries -that can map a section of executable code to a name. This interface is described in the +profiling tool (such as `perf `_ or +`samply `_). A running process may create a +file in the ``/tmp`` directory, which contains entries that can map a section +of executable code to a name. This interface is described in the `documentation of the Linux Perf tool `_. diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 96d757ac452b5e..fc4772bbccab57 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -2,34 +2,35 @@ .. _perf_profiling: -============================================== -Python support for the Linux ``perf`` profiler -============================================== +======================================================== +Python support for the ``perf map`` compatible profilers +======================================================== :author: Pablo Galindo -`The Linux perf profiler `_ -is a very powerful tool that allows you to profile and obtain -information about the performance of your application. -``perf`` also has a very vibrant ecosystem of tools -that aid with the analysis of the data that it produces. +`The Linux perf profiler `_ and +`samply `_ are powerful tools that allow you to +profile and obtain information about the performance of your application. +Both tools have vibrant ecosystems that aid with the analysis of the data they produce. -The main problem with using the ``perf`` profiler with Python applications is that -``perf`` only gets information about native symbols, that is, the names of +The main problem with using these profilers with Python applications is that +they only get information about native symbols, that is, the names of functions and procedures written in C. This means that the names and file names -of Python functions in your code will not appear in the output of ``perf``. +of Python functions in your code will not appear in the profiler output. Since Python 3.12, the interpreter can run in a special mode that allows Python -functions to appear in the output of the ``perf`` profiler. When this mode is +functions to appear in the output of compatible profilers. When this mode is enabled, the interpreter will interpose a small piece of code compiled on the -fly before the execution of every Python function and it will teach ``perf`` the +fly before the execution of every Python function and it will teach the profiler the relationship between this piece of code and the associated Python function using :doc:`perf map files <../c-api/perfmaps>`. .. note:: - Support for the ``perf`` profiler is currently only available for Linux on - select architectures. Check the output of the ``configure`` build step or + Support for profiling is available on Linux and macOS on select architectures. + Perf is available on Linux, while samply can be used on both Linux and macOS. + samply support on macOS is available starting from Python 3.15. + Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` to see if your system is supported. @@ -148,6 +149,31 @@ Instead, if we run the same experiment with ``perf`` support enabled we get: +Using the samply profiler +------------------------- + +samply is a modern profiler that can be used as an alternative to perf. +It uses the same perf map files that Python generates, making it compatible +with Python's profiling support. samply is particularly useful on macOS +where perf is not available. + +To use samply with Python, first install it following the instructions at +https://github.com/mstange/samply, then run:: + + $ samply record PYTHONPERFSUPPORT=1 python my_script.py + +This will open a web interface where you can analyze the profiling data +interactively. The advantage of samply is that it provides a modern +web-based interface for analyzing profiling data and works on both Linux +and macOS. + +On macOS, samply support requires Python 3.15 or later. Also on macOS, samply +can't profile signed Python executables due to restrictions by macOS. You can +profile with Python binaries that you've compiled yourself, or which are +unsigned or locally-signed (such as anything installed by Homebrew). In +order to attach to running processes on macOS, run ``samply setup`` once (and +every time samply is updated) to self-sign the samply binary. + How to enable ``perf`` profiling support ---------------------------------------- diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index d4c6fe0124af18..647c32656abd6d 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -1,5 +1,5 @@ import os -import sys +import sysconfig import unittest try: @@ -7,10 +7,14 @@ except ImportError: raise unittest.SkipTest("requires _testinternalcapi") +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 -if sys.platform != 'linux': - raise unittest.SkipTest('Linux only') - +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") class TestPerfMapWriting(unittest.TestCase): def test_write_perf_map_entry(self): diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py new file mode 100644 index 00000000000000..ec0ed37ffd047b --- /dev/null +++ b/Lib/test/test_samply_profiler.py @@ -0,0 +1,244 @@ +import unittest +import subprocess +import sys +import sysconfig +import os +import pathlib +from test import support +from test.support.script_helper import ( + make_script, +) +from test.support.os_helper import temp_dir + + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + +if support.check_sanitizer(address=True, memory=True, ub=True, function=True): + # gh-109580: Skip the test because it does crash randomly if Python is + # built with ASAN. + raise unittest.SkipTest("test crash randomly on ASAN/MSAN/UBSAN build") + + +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 + + +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") + + +def samply_command_works(): + try: + cmd = ["samply", "--help"] + except (subprocess.SubprocessError, OSError): + return False + + # Check that we can run a simple samply run + with temp_dir() as script_dir: + try: + output_file = script_dir + "/profile.json.gz" + cmd = ( + "samply", + "record", + "--save-only", + "--output", + output_file, + sys.executable, + "-c", + 'print("hello")', + ) + env = {**os.environ, "PYTHON_JIT": "0"} + stdout = subprocess.check_output( + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + ) + except (subprocess.SubprocessError, OSError): + return False + + if "hello" not in stdout: + return False + + return True + + +def run_samply(cwd, *args, **env_vars): + env = os.environ.copy() + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + output_file = cwd + "/profile.json.gz" + base_cmd = ( + "samply", + "record", + "--save-only", + "-o", output_file, + ) + proc = subprocess.run( + base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + if proc.returncode: + print(proc.stderr, file=sys.stderr) + raise ValueError(f"Samply failed with return code {proc.returncode}") + + import gzip + with gzip.open(output_file, mode="rt", encoding="utf-8") as f: + return f.read() + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfilerMixin: + def run_samply(self, script_dir, perf_mode, script): + raise NotImplementedError() + + def test_python_calls_appear_in_the_stack_if_perf_activated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply(script_dir, script) + + self.assertIn(f"py::foo:{script}", output) + self.assertIn(f"py::bar:{script}", output) + self.assertIn(f"py::baz:{script}", output) + + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply( + script_dir, script, activate_trampoline=False + ) + + self.assertNotIn(f"py::foo:{script}", output) + self.assertNotIn(f"py::bar:{script}", output) + self.assertNotIn(f"py::baz:{script}", output) + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfiler(unittest.TestCase, TestSamplyProfilerMixin): + def run_samply(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_samply(script_dir, sys.executable, "-Xperf", script) + return run_samply(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + + def test_pre_fork_compile(self): + code = """if 1: + import sys + import os + import sysconfig + from _testinternalcapi import ( + compile_perf_trampoline_entry, + perf_trampoline_set_persist_after_fork, + ) + + def foo_fork(): + pass + + def bar_fork(): + foo_fork() + + def foo(): + import time; time.sleep(1) + + def bar(): + foo() + + def compile_trampolines_for_all_functions(): + perf_trampoline_set_persist_after_fork(1) + for _, obj in globals().items(): + if callable(obj) and hasattr(obj, '__code__'): + compile_perf_trampoline_entry(obj.__code__) + + if __name__ == "__main__": + compile_trampolines_for_all_functions() + pid = os.fork() + if pid == 0: + print(os.getpid()) + bar_fork() + else: + bar() + """ + + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, "-Xperf", script], + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0) + self.assertNotIn("Error:", stderr) + child_pid = int(stdout.strip()) + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + perf_child_file = pathlib.Path(f"/tmp/perf-{child_pid}.map") + self.assertTrue(perf_file.exists()) + self.assertTrue(perf_child_file.exists()) + + perf_file_contents = perf_file.read_text() + self.assertIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", perf_file_contents) + + child_perf_file_contents = perf_child_file.read_text() + self.assertIn(f"py::foo_fork:{script}", child_perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) + + # Pre-compiled perf-map entries of a forked process must be + # identical in both the parent and child perf-map files. + perf_file_lines = perf_file_contents.split("\n") + for line in perf_file_lines: + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: + self.assertIn(line, child_perf_file_contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 35826bd713c0f6..745f472474cd9d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -43,6 +43,7 @@ Ray Allen Billy G. Allie Jamiel Almeida Kevin Altis +Nazım Can Altınova Samy Lahfa Skyler Leigh Amador Joe Amenta diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst new file mode 100644 index 00000000000000..470f3311526e83 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst @@ -0,0 +1,3 @@ +Add support for perf trampoline on macOS, to allow profilers wit JIT map +support to read Python calls. While profiling, ``PYTHONPERFSUPPORT=1`` can +be appended to enable the trampoline. diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index a14e68c0e81932..93adae3d99038f 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,5 +1,9 @@ .text +#if defined(__APPLE__) + .globl __Py_trampoline_func_start +#else .globl _Py_trampoline_func_start +#endif # The following assembly is equivalent to: # PyObject * # trampoline(PyThreadState *ts, _PyInterpreterFrame *f, @@ -7,7 +11,11 @@ # { # return evaluator(ts, f, throwflag); # } +#if defined(__APPLE__) +__Py_trampoline_func_start: +#else _Py_trampoline_func_start: +#endif #ifdef __x86_64__ #if defined(__CET__) && (__CET__ & 1) endbr64 @@ -35,9 +43,14 @@ _Py_trampoline_func_start: addi sp,sp,16 jr ra #endif +#if defined(__APPLE__) + .globl __Py_trampoline_func_end +__Py_trampoline_func_end: +#else .globl _Py_trampoline_func_end _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits +#endif # Note for indicating the assembly code supports CET #if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) .section .note.gnu.property,"a" diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 469882d9b2f025..3dcbcd34003102 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -66,7 +66,9 @@ #ifdef PY_HAVE_PERF_TRAMPOLINE /* Standard library includes for perf jitdump implementation */ -#include // ELF architecture constants +#if defined(__linux__) +# include // ELF architecture constants +#endif #include // File control operations #include // Standard I/O operations #include // Standard library functions @@ -74,7 +76,9 @@ #include // System data types #include // System calls (sysconf, getpid) #include // Time functions (gettimeofday) -#include // System call interface +#if defined(__linux__) +# include // System call interface +#endif // ============================================================================= // CONSTANTS AND CONFIGURATION @@ -101,6 +105,22 @@ * based on the actual unwind information requirements. */ + +/* These constants are defined inside , which we can't use outside of linux. */ +#if !defined(__linux__) +# if defined(__i386__) || defined(_M_IX86) +# define EM_386 3 +# elif defined(__arm__) || defined(_M_ARM) +# define EM_ARM 40 +# elif defined(__x86_64__) || defined(_M_X64) +# define EM_X86_64 62 +# elif defined(__aarch64__) +# define EM_AARCH64 183 +# elif defined(__riscv) +# define EM_RISCV 243 +# endif +#endif + /* Convenient access to the global trampoline API state */ #define trampoline_api _PyRuntime.ceval.perf.trampoline_api @@ -194,7 +214,7 @@ struct BaseEvent { typedef struct { struct BaseEvent base; // Common event header uint32_t process_id; // Process ID where code was generated - uint32_t thread_id; // Thread ID where code was generated + uint64_t thread_id; // Thread ID where code was generated uint64_t vma; // Virtual memory address where code is loaded uint64_t code_address; // Address of the actual machine code uint64_t code_size; // Size of the machine code in bytes @@ -1035,6 +1055,10 @@ static void* perf_map_jit_init(void) { return NULL; // Failed to get page size } +#if defined(__APPLE__) + // On macOS, samply uses a preload to find jitdumps and this mmap can be slow. + perf_jit_map_state.mapped_buffer = NULL; +#else /* * Map the first page of the jitdump file * @@ -1057,6 +1081,7 @@ static void* perf_map_jit_init(void) { close(fd); return NULL; // Memory mapping failed } +#endif perf_jit_map_state.mapped_size = page_size; @@ -1263,7 +1288,11 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev.base.size = sizeof(ev) + (name_length+1) + size; ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); +#if defined(__APPLE__) + pthread_threadid_np(NULL, &ev.thread_id); +#else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call +#endif ev.vma = base; // Virtual memory address ev.code_address = base; // Same as VMA for our use case ev.code_size = size; diff --git a/configure b/configure index 8db2e9c46abba2..74df430d10d6bd 100755 --- a/configure +++ b/configure @@ -13816,6 +13816,8 @@ case $PLATFORM_TRIPLET in #( perf_trampoline=yes ;; #( aarch64-linux-gnu) : perf_trampoline=yes ;; #( + darwin) : + perf_trampoline=yes ;; #( *) : perf_trampoline=no ;; diff --git a/configure.ac b/configure.ac index c839dd65a5fc5a..29e36f217402f1 100644 --- a/configure.ac +++ b/configure.ac @@ -3692,12 +3692,13 @@ case "$ac_sys_system" in esac AC_MSG_RESULT([$SHLIBS]) -dnl perf trampoline is Linux specific and requires an arch-specific +dnl perf trampoline is Linux and macOS specific and requires an arch-specific dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], [aarch64-linux-gnu], [perf_trampoline=yes], + [darwin], [perf_trampoline=yes], [perf_trampoline=no] ) AC_MSG_RESULT([$perf_trampoline]) From d6cf05b5d00abbfdf9008d646c14c6ef7c83b61a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 22 Jul 2025 18:15:29 +0200 Subject: [PATCH 9/9] Fix tables in 'Using on Windows' for the text writer (#137012) --- Doc/using/windows.rst | 362 +++++++++++++++++++++++++----------------- 1 file changed, 213 insertions(+), 149 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 7cc50bccb3724a..c66308e61e89da 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -385,50 +385,77 @@ The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative customization. -.. csv-table:: Standard configuration options - :header: "Config Key", "Environment Variable", "Description" - :widths: 2, 2, 4 - - ``default_tag``,``PYTHON_MANAGER_DEFAULT``,"The preferred default - version to launch or install. By default, this is interpreted as the most - recent non-prerelease version from the CPython team. - " - ``default_platform``,``PYTHON_MANAGER_DEFAULT_PLATFORM``,"The preferred - default platform to launch or install. This is treated as a suffix to the - specified tag, such that ``py -V:3.14`` would prefer an install for - ``3.14-64`` if it exists (and ``default_platform`` is ``-64``), but will use - ``3.14`` if no tagged install exists. - " - ``logs_dir``,``PYTHON_MANAGER_LOGS``,"The location where log files are - written. By default, :file:`%TEMP%`. - " - ``automatic_install``,``PYTHON_MANAGER_AUTOMATIC_INSTALL``,"True to - allow automatic installs when specifying a particular runtime to launch. - By default, true. - " - ``include_unmanaged``,``PYTHON_MANAGER_INCLUDE_UNMANAGED``,"True to - allow listing and launching runtimes that were not installed by the Python - install manager, or false to exclude them. By default, true. - " - ``shebang_can_run_anything``,"``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` - ","True to allow shebangs in ``.py`` files to launch applications other than - Python runtimes, or false to prevent it. By default, true. - " - ``log_level``,"``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG``","Set - the default level of output (0-50) By default, 20. Lower values produce more - output. The environment variables are boolean, and may produce additional - output during startup that is later suppressed by other configuration. - " - ``confirm``,``PYTHON_MANAGER_CONFIRM``,"True to confirm certain actions - before taking them (such as uninstall), or false to skip the confirmation. By - default, true. - " - ``install.source``,``PYTHON_MANAGER_SOURCE_URL``,"Override the index - feed to obtain new installs from. - " - ``list.format``,``PYTHON_MANAGER_LIST_FORMAT``,"Specify the default - format used by the ``py list`` command. By default, ``table``. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 2, 2, 4 + +.. rubric:: Standard configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Environment Variable + - Description + + * - ``default_tag`` + - ``PYTHON_MANAGER_DEFAULT`` + - The preferred default version to launch or install. + By default, this is interpreted as the most recent non-prerelease version + from the CPython team. + + * - ``default_platform`` + - ``PYTHON_MANAGER_DEFAULT_PLATFORM`` + - The preferred default platform to launch or install. + This is treated as a suffix to the specified tag, such that ``py -V:3.14`` + would prefer an install for ``3.14-64`` if it exists + (and ``default_platform`` is ``-64``), + but will use ``3.14`` if no tagged install exists. + + * - ``logs_dir`` + - ``PYTHON_MANAGER_LOGS`` + - The location where log files are written. + By default, :file:`%TEMP%`. + + * - ``automatic_install`` + - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` + - True to allow automatic installs when specifying a particular runtime + to launch. + By default, true. + + * - ``include_unmanaged`` + - ``PYTHON_MANAGER_INCLUDE_UNMANAGED`` + - True to allow listing and launching runtimes that were not installed + by the Python install manager, or false to exclude them. + By default, true. + + * - ``shebang_can_run_anything`` + - ``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` + - True to allow shebangs in ``.py`` files to launch applications other than + Python runtimes, or false to prevent it. + By default, true. + + * - ``log_level`` + - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` + - Set the default level of output (0-50). + By default, 20. + Lower values produce more output. + The environment variables are boolean, and may produce additional + output during startup that is later suppressed by other configuration. + + * - ``confirm`` + - ``PYTHON_MANAGER_CONFIRM`` + - True to confirm certain actions before taking them (such as uninstall), + or false to skip the confirmation. + By default, true. + + * - ``install.source`` + - ``PYTHON_MANAGER_SOURCE_URL`` + - Override the index feed to obtain new installs from. + + * - ``list.format`` + - ``PYTHON_MANAGER_LIST_FORMAT`` + - Specify the default format used by the ``py list`` command. + By default, ``table``. Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -492,7 +519,7 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. -.. note: +.. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no @@ -638,46 +665,64 @@ variable will be used instead. Configuration settings that are paths are interpreted as relative to the directory containing the configuration file that specified them. -.. csv-table:: Administrative configuration options - :header: "Config Key", "Description" - :widths: 1, 4 - - ``base_config``,"The highest priority configuration file to read. Note that - only the built-in configuration file and the registry can modify this - setting. - " - ``user_config``,"The second configuration file to read. - " - ``additional_config``,"The third configuration file to read. - " - ``registry_override_key``,"Registry location to check for overrides. Note - that only the built-in configuration file can modify this setting. - " - ``bundled_dir``,"Read-only directory containing locally cached files. - " - ``install.fallback_source``,"Path or URL to an index to consult when the - main index cannot be accessed. - " - ``install.enable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to allow (e.g. ``""pep514,start""``). Enabled shortcuts may still be disabled - by ``disable_shortcut_kinds``. - " - ``install.disable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to exclude (e.g. ``""pep514,start""``). Disabled shortcuts are not - reactivated by ``enable_shortcut_kinds``. - " - ``pep514_root``,"Registry location to read and write PEP 514 entries into. - By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. - " - ``start_folder``,"Start menu folder to write shortcuts into. By default, - ``Python``. This path is relative to the user's Programs folder. - " - ``virtual_env``,"Path to the active virtual environment. By default, this - is ``%VIRTUAL_ENV%``, but may be set empty to disable venv detection. - " - ``shebang_can_run_anything_silently``,"True to suppress visible warnings - when a shebang launches an application other than a Python runtime. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 4 + +.. rubric:: Administrative configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Description + + * - ``base_config`` + - The highest priority configuration file to read. + Note that only the built-in configuration file and the registry can + modify this setting. + + * - ``user_config`` + - The second configuration file to read. + + * - ``additional_config`` + - The third configuration file to read. + + * - ``registry_override_key`` + - Registry location to check for overrides. + Note that only the built-in configuration file can modify this setting. + + * - ``bundled_dir`` + - Read-only directory containing locally cached files. + + * - ``install.fallback_source`` + - Path or URL to an index to consult when the main index cannot be accessed. + + * - ``install.enable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to allow (e.g. ``"pep514,start"``). + Enabled shortcuts may still be disabled by ``disable_shortcut_kinds``. + + * - ``install.disable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to exclude + (e.g. ``"pep514,start"``). + Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + + * - ``pep514_root`` + - Registry location to read and write PEP 514 entries into. + By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. + + * - ``start_folder`` + - Start menu folder to write shortcuts into. + By default, ``Python``. + This path is relative to the user's Programs folder. + + * - ``virtual_env`` + - Path to the active virtual environment. + By default, this is ``%VIRTUAL_ENV%``, but may be set empty + to disable venv detection. + + * - ``shebang_can_run_anything_silently`` + - True to suppress visible warnings when a shebang launches an application + other than a Python runtime. .. _install-freethreaded-windows: @@ -716,70 +761,89 @@ issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). -.. csv-table:: Troubleshooting - :header: "Symptom", "Things to try" - :widths: 1, 1 - - "``python`` gives me a ""command not found"" error or opens the Store app - when I type it in my terminal.", "Did you :ref:`install the Python install - manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python (default)"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "", "Check that the ``py`` and ``pymanager`` commands work. - " - "``py`` gives me a ""command not found"" error when I type it in my - terminal.","Did you :ref:`install the Python install manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python install manager"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "``py`` gives me a ""can't open file"" error when I type commands in my - terminal.", "This usually means you have the legacy launcher installed and it - has priority over the Python install manager. To remove, click Start, open - ""Installed apps"", search for ""Python launcher"" and uninstall it. - " - "``python`` doesn't launch the same runtime as ``py``", "Click Start, open - ""Installed apps"", look for any existing Python runtimes, and either remove - them or Modify and disable the :envvar:`PATH` options. - " - "", "Click Start, open ""Manage app execution aliases"", and check that your - ``python.exe`` alias is set to ""Python (default)"" - " - "``python`` and ``py`` don't launch the runtime I expect", "Check your - ``PYTHON_MANAGER_DEFAULT`` environment variable or ``default_tag`` - configuration. The ``py list`` command will show your default based on these - settings. - " - "", "Installs that are managed by the Python install manager will be chosen - ahead of unmanaged installs. Use ``py install`` to install the runtime you - expect, or configure your default tag. - " - "", "Prerelease and experimental installs that are not managed by the Python - install manager may be chosen ahead of stable releases. Configure your - default tag or uninstall the prerelease runtime and reinstall using ``py - install``. - " - "``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or - ``py``","Click Start, open ""Manage app execution aliases"", and check that - your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your - others. - " - "``pip`` gives me a ""command not found"" error when I type it in my - terminal.","Have you activated a virtual environment? Run the - ``.venv\Scripts\activate`` script in your terminal to activate. - " - "","The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, or alternatively - the ``python -m pip install --force pip`` command will recreate the - executables and show you the path to add to :envvar:`PATH`. These scripts are - separated for each runtime, and so you may need to add multiple paths. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 1 + +.. rubric:: Troubleshooting + +.. list-table:: + :header-rows: 1 + + * - Symptom + - Things to try + + * - ``python`` gives me a "command not found" error or opens the Store app + when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Check that the ``py`` and ``pymanager`` commands work. + + * - ``py`` gives me a "command not found" error when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - ``py`` gives me a "can't open file" error when I type commands in my + terminal. + - This usually means you have the legacy launcher installed and + it has priority over the Python install manager. + To remove, click Start, open "Installed apps", + search for "Python launcher" and uninstall it. + + * - ``python`` doesn't launch the same runtime as ``py`` + - Click Start, open "Installed apps", look for any existing Python runtimes, + and either remove them or Modify and disable the :envvar:`PATH` options. + + * - + - Click Start, open "Manage app execution aliases", and check that your + ``python.exe`` alias is set to "Python (default)" + + * - ``python`` and ``py`` don't launch the runtime I expect + - Check your ``PYTHON_MANAGER_DEFAULT`` environment variable + or ``default_tag`` configuration. + The ``py list`` command will show your default based on these settings. + + * - + - Installs that are managed by the Python install manager will be chosen + ahead of unmanaged installs. + Use ``py install`` to install the runtime you expect, + or configure your default tag. + + * - + - Prerelease and experimental installs that are not managed by the Python + install manager may be chosen ahead of stable releases. + Configure your default tag or uninstall the prerelease runtime + and reinstall using ``py install``. + + * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` + - Click Start, open "Manage app execution aliases", and check that your + ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. + + * - ``pip`` gives me a "command not found" error when I type it in my terminal. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead, + or alternatively the ``python -m pip install --force pip`` command + will recreate the executables and show you the path to + add to :envvar:`PATH`. + These scripts are separated for each runtime, and so you may need to + add multiple paths. .. _windows-embeddable: