From 149bddcc216a398d71ec9497e9bf3ec03d6f2914 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 22 Jul 2025 07:51:16 +0100 Subject: [PATCH 1/4] gh-131531: Android test fixes (#136845) Modifies the test runner script to no longer export the the HOST environment variable, and to allow for tests that produce no Python output (output from the Android console is still expected and required). These changes stem from knowledge gained during developing a PR for Android support in cibuildwheel. --- Android/android.py | 73 +++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/Android/android.py b/Android/android.py index a3a48c0c6b7027..75f73cd30993da 100755 --- a/Android/android.py +++ b/Android/android.py @@ -50,7 +50,19 @@ + (".bat" if os.name == "nt" else "") ) -logcat_started = False +# Whether we've seen any output from Python yet. +python_started = False + +# Buffer for verbose output which will be displayed only if a test fails and +# there has been no output from Python. +hidden_output = [] + + +def log_verbose(context, line, stream=sys.stdout): + if context.verbose: + stream.write(line) + else: + hidden_output.append((stream, line)) def delete_glob(pattern): @@ -118,7 +130,7 @@ def android_env(host): env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( f"set -eu; " - f"export HOST={host}; " + f"HOST={host}; " f"PREFIX={prefix}; " f". {env_script}; " f"export", @@ -453,17 +465,19 @@ async def logcat_task(context, initial_devices): # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - hidden_output = [] + logcat_started = False async with async_process( *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + logcat_started = True level, message = match.groups() else: - # If the regex doesn't match, this is probably the second or - # subsequent line of a multi-line message. Python won't produce - # such messages, but other components might. + # If the regex doesn't match, this is either a logcat startup + # error, or the second or subsequent line of a multi-line + # message. Python won't produce multi-line messages, but other + # components might. level, message = None, line # Exclude high-volume messages which are rarely useful. @@ -483,25 +497,22 @@ async def logcat_task(context, initial_devices): # tag indicators from Python's stdout and stderr. for prefix in ["python.stdout: ", "python.stderr: "]: if message.startswith(prefix): - global logcat_started - logcat_started = True + global python_started + python_started = True stream.write(message.removeprefix(prefix)) break else: - if context.verbose: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - stream.write(line) - else: - hidden_output.append(line) + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + log_verbose(context, line, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. # Distinguish this from a logcat startup error by checking whether we've - # received a message from Python yet. + # received any logcat messages yet. status = await wait_for(process.wait(), timeout=1) if status != 0 and not logcat_started: - raise CalledProcessError(status, args, "".join(hidden_output)) + raise CalledProcessError(status, args) def stop_app(serial): @@ -516,16 +527,6 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected - hidden_output = [] - - def log(line): - # Gradle may take several minutes to install SDK packages, so it's worth - # showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - hidden_output.append(line) - if context.command: mode = "-c" module = context.command @@ -550,7 +551,7 @@ def log(line): ] if context.verbose >= 2: args.append("--info") - log("> " + join_command(args)) + log_verbose(context, f"> {join_command(args)}\n") try: async with async_process( @@ -558,7 +559,12 @@ def log(line): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - log(line) + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + log_verbose(context, line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -566,11 +572,6 @@ def log(line): else: raise CalledProcessError(status, args) finally: - # If logcat never started, then something has gone badly wrong, so the - # user probably wants to see the Gradle output even in non-verbose mode. - if hidden_output and not logcat_started: - sys.stdout.write("".join(hidden_output)) - # Gradle does not stop the tests when interrupted. if context.connected: stop_app(context.connected) @@ -600,6 +601,12 @@ async def run_testbed(context): except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* CalledProcessError as e: + # If Python produced no output, then the user probably wants to see the + # verbose output to explain why the test failed. + if not python_started: + for stream, line in hidden_output: + stream.write(line) + # Extract it from the ExceptionGroup so it can be handled by `main`. raise e.exceptions[0] From 9a21df7c0a494e2819775eabd522ebec994d96c0 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Tue, 22 Jul 2025 10:22:08 +0200 Subject: [PATCH 2/4] Fix 3 typos in "Next" News items (#136892) --- Misc/NEWS.d/3.14.0b1.rst | 4 ++-- .../2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst | 2 +- .../Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst | 2 +- .../2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index 041fbaf2051719..02ceb82b556386 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -1756,7 +1756,7 @@ Add support for macOS multi-arch builds with the JIT enabled .. nonce: q9fvyM .. section: Core and Builtins -PyREPL now supports syntax highlighing. Contributed by Łukasz Langa. +PyREPL now supports syntax highlighting. Contributed by Łukasz Langa. .. @@ -1797,7 +1797,7 @@ non-``None`` ``closure``. Patch by Bartosz Sławecki. .. nonce: Uj7lyY .. section: Core and Builtins -Fix a bug that was allowing newlines inconsitently in format specifiers for +Fix a bug that was allowing newlines inconsistently in format specifiers for single-quoted f-strings. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst index 5c0813b1a0abda..767d7b97726971 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst @@ -1 +1 @@ -Fix PyREPL syntax highlightning on match cases after multi-line case. Contributed by Olga Matoula. +Fix PyREPL syntax highlighting on match cases after multi-line case. Contributed by Olga Matoula. diff --git a/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst index 0a0d66ac0b8abf..ddc2310392fe92 100644 --- a/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst +++ b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst @@ -1,2 +1,2 @@ -Fix pickling failures for protocols 0 and 1 for many objects realted to +Fix pickling failures for protocols 0 and 1 for many objects related to subinterpreters. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst b/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst index 25599a865b7246..ebe3ab0e7d1993 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst @@ -1,4 +1,4 @@ The cases generator no longer accepts type annotations on stack items. -Conversions to non-default types are now done explictly in bytecodes.c and +Conversions to non-default types are now done explicitly in bytecodes.c and optimizer_bytecodes.c. This will simplify code generation for top-of-stack caching and other future features. From 22c8658906526032c24bb2f4a02fb28e2d366bcb Mon Sep 17 00:00:00 2001 From: Dave Peck Date: Tue, 22 Jul 2025 02:44:13 -0700 Subject: [PATCH 3/4] gh-132661: Document t-strings and `templatelib` (#135229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Petr Viktorin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Loïc Simon Co-authored-by: pauleveritt --- Doc/glossary.rst | 7 +- Doc/library/ast.rst | 54 ++++- Doc/library/dis.rst | 42 ++++ Doc/library/stdtypes.rst | 6 +- Doc/library/string.rst | 25 ++- Doc/library/string.templatelib.rst | 313 +++++++++++++++++++++++++++++ Doc/library/text.rst | 1 + Doc/reference/compound_stmts.rst | 4 +- Doc/reference/lexical_analysis.rst | 50 ++++- Doc/tutorial/inputoutput.rst | 9 +- 10 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 Doc/library/string.templatelib.rst diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 199a917f9f101e..b7bd547d38fd1e 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -462,7 +462,7 @@ Glossary core and with user code. f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called + String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -1322,6 +1322,11 @@ Glossary See also :term:`borrowed reference`. + t-string + String literals prefixed with ``t`` or ``T`` are commonly called + "t-strings" which is short for + :ref:`template string literals `. + text encoding A string in Python is a sequence of Unicode code points (in range ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ef6c62dca1e124..b24459b5c6346f 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -289,9 +289,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting + * 115 (``ord('s')``): ``!s`` string formatting + * 114 (``ord('r')``): ``!r`` repr formatting + * 97 (``ord('a')``): ``!a`` ASCII formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -325,6 +325,54 @@ Literals Constant(value='.3')]))])) +.. class:: TemplateStr(values) + + A t-string, comprising a series of :class:`Interpolation` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('t"{name} finished {place:ordinal}"', mode='eval'), indent=4)) + Expression( + body=TemplateStr( + values=[ + Interpolation( + value=Name(id='name'), + str='name', + conversion=-1), + Constant(value=' finished '), + Interpolation( + value=Name(id='place'), + str='place', + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='ordinal')]))])) + + .. versionadded:: 3.14 + + +.. class:: Interpolation(value, str, conversion, format_spec) + + Node representing a single interpolation field in a t-string. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``str`` is a constant containing the text of the interpolation expression. + * ``conversion`` is an integer: + + * -1: no conversion + * 115: ``!s`` string conversion + * 114: ``!r`` repr conversion + * 97: ``!a`` ascii conversion + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + .. versionadded:: 3.14 + + .. class:: List(elts, ctx) Tuple(elts, ctx) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 11685a32f48e4f..ac8a911c40a860 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1120,6 +1120,48 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: BUILD_TEMPLATE + + Constructs a new :class:`~string.templatelib.Template` from a tuple + of strings and a tuple of interpolations and pushes the resulting instance + onto the stack:: + + interpolations = STACK.pop() + strings = STACK.pop() + STACK.append(_build_template(strings, interpolations)) + + .. versionadded:: 3.14 + + +.. opcode:: BUILD_INTERPOLATION (format) + + Constructs a new :class:`~string.templatelib.Interpolation` from a + value and its source expression and pushes the resulting instance onto the + stack. + + If no conversion or format specification is present, ``format`` is set to + ``2``. + + If the low bit of ``format`` is set, it indicates that the interpolation + contains a format specification. + + If ``format >> 2`` is non-zero, it indicates that the interpolation + contains a conversion. The value of ``format >> 2`` is the conversion type + (``0`` for no conversion, ``1`` for ``!s``, ``2`` for ``!r``, and + ``3`` for ``!a``):: + + conversion = format >> 2 + if format & 1: + format_spec = STACK.pop() + else: + format_spec = None + expression = STACK.pop() + value = STACK.pop() + STACK.append(_build_interpolation(value, expression, conversion, format_spec)) + + .. versionadded:: 3.14 + + .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8e688f03eb3a87..90683c0b00d78a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2675,9 +2675,9 @@ For example: lead to a number of common errors (such as failing to display tuples and dictionaries correctly). Using the newer :ref:`formatted string literals `, the :meth:`str.format` interface, or :ref:`template strings - ` may help avoid these errors. Each of these - alternatives provides their own trade-offs and benefits of simplicity, - flexibility, and/or extensibility. + ($-strings) ` may help avoid these errors. + Each of these alternatives provides their own trade-offs and benefits of + simplicity, flexibility, and/or extensibility. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 23e15780075435..83e8ee2722ed8a 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -198,8 +198,9 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but it is -less sophisticated and, in particular, does not support arbitrary expressions. +related to that of :ref:`formatted string literals ` and +:ref:`template string literals `, but it is less sophisticated +and, in particular, does not support arbitrary expressions. .. index:: single: {} (curly brackets); in string formatting @@ -264,6 +265,8 @@ Some simple format string examples:: "Weight in tons {0.weight}" # 'weight' attribute of first positional arg "Units destroyed: {players[0]}" # First element of keyword argument 'players'. +.. _formatstrings-conversion: + The *conversion* field causes a type coercion before formatting. Normally, the job of formatting a value is done by the :meth:`~object.__format__` method of the value itself. However, in some cases it is desirable to force a type to be formatted @@ -306,7 +309,7 @@ Format Specification Mini-Language "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). +:ref:`formatstrings`, :ref:`f-strings`, and :ref:`t-strings`). They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -789,10 +792,20 @@ Nesting arguments and more complex examples:: -.. _template-strings: +.. _template-strings-pep292: -Template strings ----------------- +Template strings ($-strings) +---------------------------- + +.. note:: + + The feature described here was introduced in Python 2.4. It is unrelated + to, and should not be confused with, the newer + :ref:`template strings ` and + :ref:`t-string literal syntax ` introduced in Python 3.14. + T-string literals evaluate to instances of a different + :class:`~string.templatelib.Template` class, found in the + :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst new file mode 100644 index 00000000000000..31b90d75f411f0 --- /dev/null +++ b/Doc/library/string.templatelib.rst @@ -0,0 +1,313 @@ +:mod:`!string.templatelib` --- Support for template string literals +=================================================================== + +.. module:: string.templatelib + :synopsis: Support for template string literals. + +**Source code:** :source:`Lib/string/templatelib.py` + +-------------- + +.. seealso:: + + * :ref:`Format strings ` + * :ref:`T-string literal syntax ` + + +.. _template-strings: + +Template strings +---------------- + +.. versionadded:: 3.14 + +Template strings are a formatting mechanism that allows for deep control over +how strings are processed. You can create templates using +:ref:`t-string literal syntax `, which is identical to +:ref:`f-string syntax ` but uses a ``t`` instead of an ``f``. +While f-strings evaluate to ``str``, t-strings create a :class:`Template` +instance that gives you access to the static and interpolated (in curly braces) +parts of a string *before* they are combined. + + +.. _templatelib-template: + +Template +-------- + +The :class:`!Template` class describes the contents of a template string. + +:class:`!Template` instances are immutable: their attributes cannot be +reassigned. + +.. class:: Template(*args) + + Create a new :class:`!Template` object. + + :param args: A mix of strings and :class:`Interpolation` instances in any order. + :type args: str | Interpolation + + The most common way to create a :class:`!Template` instance is to use the + :ref:`t-string literal syntax `. This syntax is identical to that of + :ref:`f-strings ` except that it uses a ``t`` instead of an ``f``: + + >>> name = "World" + >>> template = t"Hello {name}!" + >>> type(template) + + + Templates ars stored as sequences of literal :attr:`~Template.strings` + and dynamic :attr:`~Template.interpolations`. + A :attr:`~Template.values` attribute holds the interpolation values: + + >>> template.strings + ('Hello ', '!') + >>> template.interpolations + (Interpolation('World', ...),) + >>> template.values + ('World',) + + The :attr:`!strings` tuple has one more element than :attr:`!interpolations` + and :attr:`!values`; the interpolations “belong” between the strings. + This may be easier to understand when tuples are aligned:: + + template.strings: ('Hello ', '!') + template.values: ( 'World', ) + + While literal syntax is the most common way to create :class:`!Template` + instances, it is also possible to create them directly using the constructor: + + >>> from string.templatelib import Interpolation, Template + >>> name = "World" + >>> template = Template("Hello, ", Interpolation(name, "name"), "!") + >>> list(template) + ['Hello, ', Interpolation('World', 'name', None, ''), '!'] + + If two or more consecutive strings are passed, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: + + >>> from string.templatelib import Template + >>> template = Template("Hello ", "World", "!") + >>> template.strings + ('Hello World!',) + + If two or more consecutive interpolations are passed, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: + + >>> from string.templatelib import Interpolation, Template + >>> template = Template(Interpolation("World", "name"), Interpolation("!", "punctuation")) + >>> template.strings + ('', '', '') + + .. attribute:: strings + :type: tuple[str, ...] + + A :ref:`tuple ` of the static strings in the template. + + >>> name = "World" + >>> t"Hello {name}!".strings + ('Hello ', '!') + + Empty strings *are* included in the tuple: + + >>> name = "World" + >>> t"Hello {name}{name}!".strings + ('Hello ', '', '!') + + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: + + >>> t"".strings + ('',) + >>> t"".values + () + >>> t"{'cheese'}".strings + ('', '') + >>> t"{'cheese'}".values + ('cheese',) + + .. attribute:: interpolations + :type: tuple[Interpolation, ...] + + A tuple of the interpolations in the template. + + >>> name = "World" + >>> t"Hello {name}!".interpolations + (Interpolation('World', 'name', None, ''),) + + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: + + >>> t"Hello!".interpolations + () + + .. attribute:: values + :type: tuple[Any, ...] + + A tuple of all interpolated values in the template. + + >>> name = "World" + >>> t"Hello {name}!".values + ('World',) + + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is equivalent to + ``tuple(i.value for i in template.interpolations)``. + + .. describe:: iter(template) + + Iterate over the template, yielding each string and + :class:`Interpolation` in order. + + >>> name = "World" + >>> list(t"Hello {name}!") + ['Hello ', Interpolation('World', 'name', None, ''), '!'] + + Empty strings are *not* included in the iteration: + + >>> name = "World" + >>> list(t"Hello {name}{name}") + ['Hello ', Interpolation('World', 'name', None, ''), Interpolation('World', 'name', None, '')] + + .. describe:: template + other + template += other + + Concatenate this template with another, returning a new + :class:`!Template` instance: + + >>> name = "World" + >>> list(t"Hello " + t"there {name}!") + ['Hello there ', Interpolation('World', 'name', None, ''), '!'] + + Concatenation between a :class:`!Template` and a ``str`` is *not* supported. + This is because it is ambiguous whether the string should be treated as + a static string or an interpolation. If you want to concatenate a + :class:`!Template` with a string, you should either wrap the string + directly in a :class:`!Template` (to treat it as a static string) or use + an :class:`!Interpolation` (to treat it as dynamic): + + >>> from string.templatelib import Template, Interpolation + >>> template = t"Hello " + >>> # Treat "there " as a static string + >>> template += Template("there ") + >>> # Treat name as an interpolation + >>> name = "World" + >>> template += Template(Interpolation(name, "name")) + >>> list(template) + ['Hello there ', Interpolation('World', 'name', None, '')] + + +.. class:: Interpolation(value, expression="", conversion=None, format_spec="") + + Create a new :class:`!Interpolation` object. + + :param value: The evaluated, in-scope result of the interpolation. + :type value: object + + :param expression: The text of a valid Python expression, or an empty string. + :type expression: str + + :param conversion: The optional :ref:`conversion ` to be used, one of r, s, and a. + :type conversion: ``Literal["a", "r", "s"] | None`` + + :param format_spec: An optional, arbitrary string used as the :ref:`format specification ` to present the value. + :type format_spec: str + + The :class:`!Interpolation` type represents an expression inside a template string. + + :class:`!Interpolation` instances are immutable: their attributes cannot be + reassigned. + + .. attribute:: value + + :returns: The evaluated value of the interpolation. + :type: object + + >>> t"{1 + 2}".interpolations[0].value + 3 + + .. attribute:: expression + + :returns: The text of a valid Python expression, or an empty string. + :type: str + + The :attr:`~Interpolation.expression` is the original text of the + interpolation's Python expression, if the interpolation was created + from a t-string literal. Developers creating interpolations manually + should either set this to an empty string or choose a suitable valid + Python expression. + + >>> t"{1 + 2}".interpolations[0].expression + '1 + 2' + + .. attribute:: conversion + + :returns: The conversion to apply to the value, or ``None``. + :type: ``Literal["a", "r", "s"] | None`` + + The :attr:`!Interpolation.conversion` is the optional conversion to apply + to the value: + + >>> t"{1 + 2!a}".interpolations[0].conversion + 'a' + + .. note:: + + Unlike f-strings, where conversions are applied automatically, + the expected behavior with t-strings is that code that *processes* the + :class:`!Template` will decide how to interpret and whether to apply + the :attr:`!Interpolation.conversion`. + + .. attribute:: format_spec + + :returns: The format specification to apply to the value. + :type: str + + The :attr:`!Interpolation.format_spec` is an optional, arbitrary string + used as the format specification to present the value: + + >>> t"{1 + 2:.2f}".interpolations[0].format_spec + '.2f' + + .. note:: + + Unlike f-strings, where format specifications are applied automatically + via the :func:`format` protocol, the expected behavior with + t-strings is that code that *processes* the :class:`!Template` will + decide how to interpret and whether to apply the format specification. + As a result, :attr:`!Interpolation.format_spec` values in + :class:`!Template` instances can be arbitrary strings, even those that + do not necessarily conform to the rules of Python's :func:`format` + protocol. + + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = Interpolation(3.0, "1 + 2", None, ".2f") + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec) + ... + 3.0 1 + 2 None .2f + + +Helper functions +---------------- + +.. function:: convert(obj, /, conversion) + + Applies formatted string literal :ref:`conversion ` + semantics to the given object *obj*. + This is frequently useful for custom template string processing logic. + + Three conversion flags are currently supported: + + * ``'s'`` which calls :func:`str` on the value, + * ``'r'`` which calls :func:`repr`, and + * ``'a'`` which calls :func:`ascii`. + + If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/text.rst b/Doc/library/text.rst index 47b678434fc899..92e7dd9a53b80d 100644 --- a/Doc/library/text.rst +++ b/Doc/library/text.rst @@ -16,6 +16,7 @@ Python's built-in string type in :ref:`textseq`. .. toctree:: string.rst + string.templatelib.rst re.rst difflib.rst textwrap.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e95fa3a6424e23..a416cbb4cc8eab 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -852,8 +852,8 @@ A literal pattern corresponds to most The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are -supported. Raw strings and byte strings are supported. :ref:`f-strings` are -not supported. +supported. Raw strings and byte strings are supported. :ref:`f-strings` +and :ref:`t-strings` are not supported. The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are for expressing :ref:`complex numbers `; they require a real number diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 567c70111c20ec..a7f8e5392b7e71 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -561,9 +561,9 @@ escapes are not treated specially. single: f'; formatted string literal single: f"; formatted string literal -A string literal with ``'f'`` or ``'F'`` in its prefix is a -:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be -combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw +A string literal with ``f`` or ``F`` in its prefix is a +:dfn:`formatted string literal`; see :ref:`f-strings`. The ``f`` may be +combined with ``r``, but not with ``b`` or ``u``, therefore raw formatted strings are possible, but formatted bytes literals are not. In triple-quoted literals, unescaped newlines and quotes are allowed (and are @@ -756,7 +756,7 @@ f-strings .. versionadded:: 3.6 A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``'f'`` or ``'F'``. These strings may contain +that is prefixed with ``f`` or ``F``. These strings may contain replacement fields, which are expressions delimited by curly braces ``{}``. While other string literals always have a constant value, formatted strings are really expressions evaluated at run time. @@ -913,6 +913,48 @@ See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. +.. _t-strings: +.. _template-string-literals: + +t-strings +--------- + +.. versionadded:: 3.14 + +A :dfn:`template string literal` or :dfn:`t-string` is a string literal +that is prefixed with ``t`` or ``T``. These strings follow the same +syntax and evaluation rules as :ref:`formatted string literals `, with +the following differences: + +- Rather than evaluating to a ``str`` object, t-strings evaluate to a + :class:`~string.templatelib.Template` object from the + :mod:`string.templatelib` module. + +- The :func:`format` protocol is not used. Instead, the format specifier and + conversions (if any) are passed to a new :class:`~string.templatelib.Interpolation` + object that is created for each evaluated expression. It is up to code that + processes the resulting :class:`~string.templatelib.Template` object to + decide how to handle format specifiers and conversions. + +- Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the expression ``{precision}`` before setting the ``format_spec`` + attribute of the resulting :class:`!Interpolation` object; if ``precision`` + is (for example) ``2``, the resulting format specifier will be ``'.2f'``. + +- When the equal sign ``'='`` is provided in an interpolation expression, the + resulting :class:`~string.templatelib.Template` object will have the expression + text along with a ``'='`` character placed in its + :attr:`~string.templatelib.Template.strings` attribute. The + :attr:`~string.templatelib.Template.interpolations` attribute will also + contain an ``Interpolation`` instance for the expression. By default, the + :attr:`~string.templatelib.Interpolation.conversion` attribute will be set to + ``'r'`` (that is, :func:`repr`), unless there is a conversion explicitly + specified (in which case it overrides the default) or a format specifier is + provided (in which case, the ``conversion`` defaults to ``None``). + + .. _numbers: Numeric literals diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 35b8c7cd8eb049..ea546c6a29df44 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,10 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module contains a :class:`~string.Template` class that offers -yet another way to substitute values into strings, using placeholders like -``$x`` and replacing them with values from a dictionary, but offers much less -control of the formatting. +The :mod:`string` module also contains support for so-called +:ref:`$-strings ` that offer yet another way to +substitute values into strings, using placeholders like ``$x`` and replacing +them with values from a dictionary. This syntax is easy to use, although +it offers much less control of the formatting. .. index:: single: formatted string literal From c933a6bb329bb97bc7e448388dad1b74f7ca4baa Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 22 Jul 2025 12:13:38 +0200 Subject: [PATCH 4/4] gh-124621: Emscripten: Support pyrepl in browser (GH-136931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Basic support for pyrepl in Emscripten. Limitations: * requires JSPI * no signal handling implemented As followup work, it would be nice to implement a webworker variant for when JSPI is not available and proper signal handling. Because it requires JSPI, it doesn't work in Safari. Firefox requires setting an experimental flag. All the Chromiums have full support since May. Until we make it work without JSPI, let's keep the original web_example around. Co-authored-by: Łukasz Langa Co-authored-by: Éric --- Lib/_pyrepl/trace.py | 27 ++- Lib/test/test_fcntl.py | 5 +- Makefile.pre.in | 54 +++-- ...-07-21-16-10-24.gh-issue-124621.wyoWc1.rst | 1 + Modules/clinic/posixmodule.c.h | 80 +++++++- Modules/posixmodule.c | 20 ++ Python/emscripten_syscalls.c | 116 ++++++++++- .../emscripten/config.site-wasm32-emscripten | 1 - .../{web_example => }/wasm_assets.py | 3 +- .../web_example_pyrepl_jspi/index.html | 34 +++ .../web_example_pyrepl_jspi/src.mjs | 194 ++++++++++++++++++ configure | 4 +- configure.ac | 4 +- 13 files changed, 504 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst rename Tools/wasm/emscripten/{web_example => }/wasm_assets.py (98%) create mode 100644 Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html create mode 100644 Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index a8eb2433cd3cce..943ee12f964b29 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys # types if False: @@ -12,10 +13,22 @@ trace_file = open(trace_filename, "a") -def trace(line: str, *k: object, **kw: object) -> None: - if trace_file is None: - return - if k or kw: - line = line.format(*k, **kw) - trace_file.write(line + "\n") - trace_file.flush() + +if sys.platform == "emscripten": + from posix import _emscripten_log + + def trace(line: str, *k: object, **kw: object) -> None: + if "PYREPL_TRACE" not in os.environ: + return + if k or kw: + line = line.format(*k, **kw) + _emscripten_log(line) + +else: + def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 7140a7b4f29188..222b69a6d250cd 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -8,7 +8,7 @@ import sys import unittest from test.support import ( - cpython_only, get_pagesize, is_apple, requires_subprocess, verbose + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose, is_emscripten ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink, make_bad_fd @@ -211,6 +211,7 @@ def test_fcntl_f_getpath(self): @unittest.skipUnless( hasattr(fcntl, "F_SETPIPE_SZ") and hasattr(fcntl, "F_GETPIPE_SZ"), "F_SETPIPE_SZ and F_GETPIPE_SZ are not available on all platforms.") + @unittest.skipIf(is_emscripten, "Emscripten pipefs doesn't support these") def test_fcntl_f_pipesize(self): test_pipe_r, test_pipe_w = os.pipe() try: @@ -265,12 +266,14 @@ def _check_fcntl_not_mutate_len(self, nbytes=None): @unittest.skipUnless( hasattr(fcntl, "F_SETOWN_EX") and hasattr(fcntl, "F_GETOWN_EX"), "requires F_SETOWN_EX and F_GETOWN_EX") + @unittest.skipIf(is_emscripten, "Emscripten doesn't actually support these") def test_fcntl_small_buffer(self): self._check_fcntl_not_mutate_len() @unittest.skipUnless( hasattr(fcntl, "F_SETOWN_EX") and hasattr(fcntl, "F_GETOWN_EX"), "requires F_SETOWN_EX and F_GETOWN_EX") + @unittest.skipIf(is_emscripten, "Emscripten doesn't actually support these") def test_fcntl_large_buffer(self): self._check_fcntl_not_mutate_len(2024) diff --git a/Makefile.pre.in b/Makefile.pre.in index 959ccb891f283c..fa17f5d7bfc0ac 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -804,7 +804,7 @@ build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods .PHONY: build_emscripten -build_emscripten: build_wasm web_example +build_emscripten: build_wasm web_example web_example_pyrepl_jspi # Check that the source is clean when building out of source. .PHONY: check-clean-src @@ -1095,26 +1095,28 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm32-emscripten browser web example -WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ +EMSCRIPTEN_DIR=$(srcdir)/Tools/wasm/emscripten +WEBEX_DIR=$(EMSCRIPTEN_DIR)/web_example/ + +ZIP_STDLIB=python$(VERSION)$(ABI_THREAD).zip +$(ZIP_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(EMSCRIPTEN_DIR)/wasm_assets.py \ + Makefile pybuilddir.txt Modules/Setup.local + $(PYTHON_FOR_BUILD) $(EMSCRIPTEN_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ + web_example/index.html: $(WEBEX_DIR)/index.html @mkdir -p web_example @cp $< $@ -web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs +web_example/server.py: $(WEBEX_DIR)/server.py @mkdir -p web_example @cp $< $@ -web_example/server.py: $(WEBEX_DIR)/server.py +web_example/$(ZIP_STDLIB): $(ZIP_STDLIB) @mkdir -p web_example @cp $< $@ -WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip -$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(WEBEX_DIR)/wasm_assets.py \ - Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ - --buildroot . --prefix $(prefix) -o $@ - web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ echo "Can only build web_example when target is Emscripten" ;\ @@ -1124,7 +1126,35 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.wasm web_example/python.wasm .PHONY: web_example -web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py $(WEB_STDLIB) +web_example: web_example/python.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) + +WEBEX2=web_example_pyrepl_jspi +WEBEX2_DIR=$(EMSCRIPTEN_DIR)/$(WEBEX2)/ + +$(WEBEX2)/python.mjs $(WEBEX2)/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + @mkdir -p $(WEBEX2) + @cp python.mjs $(WEBEX2)/python.mjs + @cp python.wasm $(WEBEX2)/python.wasm + +$(WEBEX2)/index.html: $(WEBEX2_DIR)/index.html + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/src.mjs: $(WEBEX2_DIR)/src.mjs + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p $(WEBEX2) + @cp $< $@ + +.PHONY: web_example_pyrepl_jspi +web_example_pyrepl_jspi: $(WEBEX2)/python.mjs $(WEBEX2)/index.html $(WEBEX2)/src.mjs $(WEBEX2)/$(ZIP_STDLIB) + ############################################################################ # Header files diff --git a/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst b/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst new file mode 100644 index 00000000000000..34049183649271 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst @@ -0,0 +1 @@ +pyrepl now works in Emscripten. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 65f5f8c9267b6c..0a281cbe6c57a2 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -12769,6 +12769,80 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(__EMSCRIPTEN__) */ +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_log__doc__, +"_emscripten_log($module, /, arg)\n" +"--\n" +"\n" +"Log something to the JS console. Emscripten only."); + +#define OS__EMSCRIPTEN_LOG_METHODDEF \ + {"_emscripten_log", _PyCFunction_CAST(os__emscripten_log), METH_FASTCALL|METH_KEYWORDS, os__emscripten_log__doc__}, + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg); + +static PyObject * +os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_emscripten_log", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_emscripten_log", "argument 'arg'", "str", args[0]); + goto exit; + } + Py_ssize_t arg_length; + arg = PyUnicode_AsUTF8AndSize(args[0], &arg_length); + if (arg == NULL) { + goto exit; + } + if (strlen(arg) != (size_t)arg_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = os__emscripten_log_impl(module, arg); + +exit: + return return_value; +} + +#endif /* defined(__EMSCRIPTEN__) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13440,4 +13514,8 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ -/*[clinic end generated code: output=6cfddb3b77dc7a40 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_LOG_METHODDEF + #define OS__EMSCRIPTEN_LOG_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ +/*[clinic end generated code: output=608e9bc5f631f688 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 47eaf5cd428a53..77622fbc4e8065 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16971,6 +16971,25 @@ os__emscripten_debugger_impl(PyObject *module) emscripten_debugger(); Py_RETURN_NONE; } + +EM_JS(void, emscripten_log_impl_js, (const char* arg), { + console.warn(UTF8ToString(arg)); +}); + +/*[clinic input] +os._emscripten_log + arg: str + +Log something to the JS console. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg) +/*[clinic end generated code: output=9749e5e293c42784 input=350aa1f70bc1e905]*/ +{ + emscripten_log_impl_js(arg); + Py_RETURN_NONE; +} #endif /* __EMSCRIPTEN__ */ @@ -17190,6 +17209,7 @@ static PyMethodDef posix_methods[] = { OS__IS_INPUTHOOK_INSTALLED_METHODDEF OS__CREATE_ENVIRON_METHODDEF OS__EMSCRIPTEN_DEBUGGER_METHODDEF + OS__EMSCRIPTEN_LOG_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index d3eedad30e3639..404d98d492a655 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -1,4 +1,5 @@ #include "emscripten.h" +#include "stdio.h" // If we're running in node, report the UID of the user in the native system as // the UID of the user. Since the nodefs will report the uid correctly, if we @@ -40,7 +41,7 @@ int __syscall_umask(int mask) { #include #include -#undef errno +#include // Variant of EM_JS that does C preprocessor substitution on the body #define EM_JS_MACROS(ret, func_name, args, body...) \ @@ -100,7 +101,7 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { return; } const origResolveGlobalSymbol = resolveGlobalSymbol; - if (!Module.onExit && globalThis?.process?.exit) { + if (ENVIRONMENT_IS_NODE && !Module.onExit) { Module.onExit = (code) => process.exit(code); } // * wrap the main symbol with WebAssembly.promising, @@ -115,7 +116,7 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { orig.sym = (...args) => { (async () => { const ret = await main(...args); - process?.exit?.(ret); + Module.onExit?.(ret); })(); _emscripten_exit_with_live_runtime(); }; @@ -185,7 +186,7 @@ EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( if (e.name !== 'ErrnoError') { throw e; } - return e.errno; + return e["errno"]; } })(); }; @@ -199,16 +200,16 @@ __wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs, // Take a promise that resolves to __wasi_errno_t and suspend until it resolves, // get the output. -EM_JS(__wasi_errno_t, __block_for_errno, (__externref_t p), { +EM_JS(int, __block_for_int, (__externref_t p), { return p; } if (WebAssembly.Suspending) { - __block_for_errno = new WebAssembly.Suspending(__block_for_errno); + __block_for_int = new WebAssembly.Suspending(__block_for_int); } ) // Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned -// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_errno +// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_int // to get the result. __wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, size_t iovs_len, __wasi_size_t *nread) { @@ -216,6 +217,103 @@ __wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, if (__builtin_wasm_ref_is_null_extern(p)) { return __wasi_fd_read_orig(fd, iovs, iovs_len, nread); } - __wasi_errno_t res = __block_for_errno(p); - return res; + return __block_for_int(p); +} + +#include +#define POLLFD_FD 0 +#define POLLFD_EVENTS 4 +#define POLLFD_REVENTS 6 +#define POLLFD_SIZE 8 +_Static_assert(offsetof(struct pollfd, fd) == 0, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, events) == 4, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct layout"); +_Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout"); + +EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), { + if (!WebAssembly.promising) { + return null; + } + return (async function() { + try { + var nonzero = 0; + var promises = []; + for (var i = 0; i < nfds; i++) { + var pollfd = fds + POLLFD_SIZE * i; + var fd = HEAP32[(pollfd + POLLFD_FD)/4]; + var events = HEAP16[(pollfd + POLLFD_EVENTS)/2]; + var mask = POLLNVAL; + var stream = FS.getStream(fd); + if (stream) { + mask = POLLIN | POLLOUT; + if (stream.stream_ops.pollAsync) { + promises.push(stream.stream_ops.pollAsync(stream, timeout).then((mask) => { + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + })); + } else if (stream.stream_ops.poll) { + var mask = stream.stream_ops.poll(stream, timeout); + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + } + } + } + await Promise.all(promises); + return nonzero; + } catch(e) { + if (e?.name !== "ErrnoError") throw e; + return -e["errno"]; + } + })(); +}); + +// Bind original poll syscall to syscall_poll_orig(). +int syscall_poll_orig(intptr_t fds, int nfds, int timeout) + __attribute__((__import_module__("env"), + __import_name__("__syscall_poll"), __warn_unused_result__)); + +int __syscall_poll(intptr_t fds, int nfds, int timeout) { + __externref_t p = __maybe_poll_async(fds, nfds, timeout); + if (__builtin_wasm_ref_is_null_extern(p)) { + return syscall_poll_orig(fds, nfds, timeout); + } + return __block_for_int(p); +} + +#include + +int syscall_ioctl_orig(int fd, int request, void* varargs) + __attribute__((__import_module__("env"), + __import_name__("__syscall_ioctl"), __warn_unused_result__)); + +int __syscall_ioctl(int fd, int request, void* varargs) { + if (request == FIOCLEX || request == FIONCLEX) { + return 0; + } + if (request == FIONBIO) { + // Implement FIONBIO via fcntl. + // TODO: Upstream this. + int flags = fcntl(fd, F_GETFL, 0); + int nonblock = **((int**)varargs); + if (flags < 0) { + return errno; + } + if (nonblock) { + flags |= O_NONBLOCK; + } else { + flags &= (~O_NONBLOCK); + } + int res = fcntl(fd, F_SETFL, flags); + if (res < 0) { + return errno; + } + return res; + } + return syscall_ioctl_orig(fd, request, varargs); } diff --git a/Tools/wasm/emscripten/config.site-wasm32-emscripten b/Tools/wasm/emscripten/config.site-wasm32-emscripten index 8c3a338dacb2dc..9f98e3f3c3bb1f 100644 --- a/Tools/wasm/emscripten/config.site-wasm32-emscripten +++ b/Tools/wasm/emscripten/config.site-wasm32-emscripten @@ -69,7 +69,6 @@ ac_cv_func_posix_fallocate=no # Syscalls that resulted in a segfault ac_cv_func_utimensat=no -ac_cv_header_sys_ioctl_h=no # sockets are supported, but only AF_INET / AF_INET6 in non-blocking mode. # Disable AF_UNIX and AF_PACKET support, see socketmodule.h. diff --git a/Tools/wasm/emscripten/web_example/wasm_assets.py b/Tools/wasm/emscripten/wasm_assets.py similarity index 98% rename from Tools/wasm/emscripten/web_example/wasm_assets.py rename to Tools/wasm/emscripten/wasm_assets.py index deeb9229a4412b..b08e7ce1114a4a 100755 --- a/Tools/wasm/emscripten/web_example/wasm_assets.py +++ b/Tools/wasm/emscripten/wasm_assets.py @@ -18,7 +18,7 @@ from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parents[4].absolute() +SRCDIR = pathlib.Path(__file__).parents[3].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -84,7 +84,6 @@ "_json": ["json/"], "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], "pyexpat": ["xml/", "xmlrpc/"], - "readline": ["rlcompleter.py"], "_sqlite3": ["sqlite3/"], "_ssl": ["ssl.py"], "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], diff --git a/Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html b/Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html new file mode 100644 index 00000000000000..1f72bd24e79a04 --- /dev/null +++ b/Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html @@ -0,0 +1,34 @@ + + + + + + + +
+ + + diff --git a/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs new file mode 100644 index 00000000000000..5642372c9d2472 --- /dev/null +++ b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs @@ -0,0 +1,194 @@ +// Much of this is adapted from here: +// https://github.com/mame/xterm-pty/blob/main/emscripten-pty.js +// Thanks to xterm-pty for making this possible! + +import createEmscriptenModule from "./python.mjs"; +import { openpty } from "https://unpkg.com/xterm-pty/index.mjs"; +import "https://unpkg.com/@xterm/xterm/lib/xterm.js"; + +var term = new Terminal(); +term.open(document.getElementById("terminal")); +const { master, slave: PTY } = openpty(); +term.loadAddon(master); +globalThis.PTY = PTY; + +async function setupStdlib(Module) { + const versionInt = Module.HEAPU32[Module._Py_Version >>> 2]; + const major = (versionInt >>> 24) & 0xff; + const minor = (versionInt >>> 16) & 0xff; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); +} + +const tty_ops = { + ioctl_tcgets: () => { + const termios = PTY.ioctl("TCGETS"); + const data = { + c_iflag: termios.iflag, + c_oflag: termios.oflag, + c_cflag: termios.cflag, + c_lflag: termios.lflag, + c_cc: termios.cc, + }; + return data; + }, + + ioctl_tcsets: (_tty, _optional_actions, data) => { + PTY.ioctl("TCSETS", { + iflag: data.c_iflag, + oflag: data.c_oflag, + cflag: data.c_cflag, + lflag: data.c_lflag, + cc: data.c_cc, + }); + return 0; + }, + + ioctl_tiocgwinsz: () => PTY.ioctl("TIOCGWINSZ").reverse(), + + get_char: () => { + throw new Error("Should not happen"); + }, + put_char: () => { + throw new Error("Should not happen"); + }, + + fsync: () => {}, +}; + +const POLLIN = 1; +const POLLOUT = 4; + +const waitResult = { + READY: 0, + SIGNAL: 1, + TIMEOUT: 2, +}; + +function onReadable() { + var handle; + var promise = new Promise((resolve) => { + handle = PTY.onReadable(() => resolve(waitResult.READY)); + }); + return [promise, handle]; +} + +function onSignal() { + // TODO: signal handling + var handle = { dispose() {} }; + var promise = new Promise((resolve) => {}); + return [promise, handle]; +} + +function onTimeout(timeout) { + var id; + var promise = new Promise((resolve) => { + if (timeout > 0) { + id = setTimeout(resolve, timeout, waitResult.TIMEOUT); + } + }); + var handle = { + dispose() { + if (id) { + clearTimeout(id); + } + }, + }; + return [promise, handle]; +} + +async function waitForReadable(timeout) { + let p1, p2, p3; + let h1, h2, h3; + try { + [p1, h1] = onReadable(); + [p2, h2] = onTimeout(timeout); + [p3, h3] = onSignal(); + return await Promise.race([p1, p2, p3]); + } finally { + h1.dispose(); + h2.dispose(); + h3.dispose(); + } +} + +const FIONREAD = 0x541b; + +const tty_stream_ops = { + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + let readBytes = PTY.read(length); + if (length && !readBytes.length) { + const status = await waitForReadable(-1); + if (status === waitResult.READY) { + readBytes = PTY.read(length); + } else { + throw new Error("Not implemented"); + } + } + buffer.set(readBytes, offset); + return readBytes.length; + }, + + write: (stream, buffer, offset, length) => { + // Note: default `buffer` is for some reason `HEAP8` (signed), while we want unsigned `HEAPU8`. + buffer = new Uint8Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength, + ); + const toWrite = Array.from(buffer.subarray(offset, offset + length)); + PTY.write(toWrite); + return length; + }, + + async pollAsync(stream, timeout) { + if (!PTY.readable && timeout) { + await waitForReadable(timeout); + } + return (PTY.readable ? POLLIN : 0) | (PTY.writable ? POLLOUT : 0); + }, + ioctl(stream, request, varargs) { + if (request === FIONREAD) { + const res = PTY.fromLdiscToUpperBuffer.length; + Module.HEAPU32[varargs / 4] = res; + return 0; + } + throw new Error("Unimplemented ioctl request"); + }, +}; + +async function setupStdio(Module) { + Object.assign(Module.TTY.default_tty_ops, tty_ops); + Object.assign(Module.TTY.stream_ops, tty_stream_ops); +} + +const emscriptenSettings = { + async preRun(Module) { + Module.addRunDependency("pre-run"); + Module.ENV.TERM = "xterm-256color"; + // Uncomment next line to turn on tracing (messages go to browser console). + // Module.ENV.PYREPL_TRACE = "1"; + + // Leak module so we can try to show traceback if we crash on startup + globalThis.Module = Module; + await Promise.all([setupStdlib(Module), setupStdio(Module)]); + Module.removeRunDependency("pre-run"); + }, +}; + +try { + await createEmscriptenModule(emscriptenSettings); +} catch (e) { + // Show JavaScript exception and traceback + console.warn(e); + // Show Python exception and traceback + Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + process.exit(1); +} diff --git a/configure b/configure index ef47f9b0df73a8..8db2e9c46abba2 100755 --- a/configure +++ b/configure @@ -9603,7 +9603,7 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY" as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" @@ -31180,9 +31180,7 @@ case $ac_sys_system in #( - py_cv_module_fcntl=n/a py_cv_module_readline=n/a - py_cv_module_termios=n/a py_cv_module_=n/a ;; #( diff --git a/configure.ac b/configure.ac index 23ed9cd35bc94b..c839dd65a5fc5a 100644 --- a/configure.ac +++ b/configure.ac @@ -2335,7 +2335,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) dnl Avoid bugs in JS fallback string decoding path @@ -7768,9 +7768,7 @@ AS_CASE([$ac_sys_system], ) dnl fcntl, readline, and termios are not particularly useful in browsers. PY_STDLIB_MOD_SET_NA( - [fcntl], [readline], - [termios], ) ], [WASI], [