Skip to content

Commit a74696a

Browse files
brettcannonbrettcannon
authored
Merge branch 'main' into is_wasm
2 parents d59b218 + d6cf05b commit a74696a

File tree

78 files changed

+2039
-708
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+2039
-708
lines changed

‎Android/android.py

Lines changed: 40 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,19 @@
5050
+ (".bat" if os.name == "nt" else "")
5151
)
5252

53-
logcat_started = False
53+
# Whether we've seen any output from Python yet.
54+
python_started = False
55+
56+
# Buffer for verbose output which will be displayed only if a test fails and
57+
# there has been no output from Python.
58+
hidden_output = []
59+
60+
61+
def log_verbose(context, line, stream=sys.stdout):
62+
if context.verbose:
63+
stream.write(line)
64+
else:
65+
hidden_output.append((stream, line))
5466

5567

5668
def delete_glob(pattern):
@@ -118,7 +130,7 @@ def android_env(host):
118130
env_script = ANDROID_DIR / "android-env.sh"
119131
env_output = subprocess.run(
120132
f"set -eu; "
121-
f"export HOST={host}; "
133+
f"HOST={host}; "
122134
f"PREFIX={prefix}; "
123135
f". {env_script}; "
124136
f"export",
@@ -453,17 +465,19 @@ async def logcat_task(context, initial_devices):
453465

454466
# `--pid` requires API level 24 or higher.
455467
args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"]
456-
hidden_output = []
468+
logcat_started = False
457469
async with async_process(
458470
*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
459471
) as process:
460472
while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
461473
if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL):
474+
logcat_started = True
462475
level, message = match.groups()
463476
else:
464-
# If the regex doesn't match, this is probably the second or
465-
# subsequent line of a multi-line message. Python won't produce
466-
# such messages, but other components might.
477+
# If the regex doesn't match, this is either a logcat startup
478+
# error, or the second or subsequent line of a multi-line
479+
# message. Python won't produce multi-line messages, but other
480+
# components might.
467481
level, message = None, line
468482

469483
# Exclude high-volume messages which are rarely useful.
@@ -483,25 +497,22 @@ async def logcat_task(context, initial_devices):
483497
# tag indicators from Python's stdout and stderr.
484498
for prefix in ["python.stdout: ", "python.stderr: "]:
485499
if message.startswith(prefix):
486-
global logcat_started
487-
logcat_started = True
500+
global python_started
501+
python_started = True
488502
stream.write(message.removeprefix(prefix))
489503
break
490504
else:
491-
if context.verbose:
492-
# Non-Python messages add a lot of noise, but they may
493-
# sometimes help explain a failure.
494-
stream.write(line)
495-
else:
496-
hidden_output.append(line)
505+
# Non-Python messages add a lot of noise, but they may
506+
# sometimes help explain a failure.
507+
log_verbose(context, line, stream)
497508

498509
# If the device disconnects while logcat is running, which always
499510
# happens in --managed mode, some versions of adb return non-zero.
500511
# Distinguish this from a logcat startup error by checking whether we've
501-
# received a message from Python yet.
512+
# received any logcat messages yet.
502513
status = await wait_for(process.wait(), timeout=1)
503514
if status != 0 and not logcat_started:
504-
raise CalledProcessError(status, args, "".join(hidden_output))
515+
raise CalledProcessError(status, args)
505516

506517

507518
def stop_app(serial):
@@ -516,16 +527,6 @@ async def gradle_task(context):
516527
task_prefix = "connected"
517528
env["ANDROID_SERIAL"] = context.connected
518529

519-
hidden_output = []
520-
521-
def log(line):
522-
# Gradle may take several minutes to install SDK packages, so it's worth
523-
# showing those messages even in non-verbose mode.
524-
if context.verbose or line.startswith('Preparing "Install'):
525-
sys.stdout.write(line)
526-
else:
527-
hidden_output.append(line)
528-
529530
if context.command:
530531
mode = "-c"
531532
module = context.command
@@ -550,27 +551,27 @@ def log(line):
550551
]
551552
if context.verbose >= 2:
552553
args.append("--info")
553-
log("> " + join_command(args))
554+
log_verbose(context, f"> {join_command(args)}\n")
554555

555556
try:
556557
async with async_process(
557558
*args, cwd=TESTBED_DIR, env=env,
558559
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
559560
) as process:
560561
while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
561-
log(line)
562+
# Gradle may take several minutes to install SDK packages, so
563+
# it's worth showing those messages even in non-verbose mode.
564+
if line.startswith('Preparing "Install'):
565+
sys.stdout.write(line)
566+
else:
567+
log_verbose(context, line)
562568

563569
status = await wait_for(process.wait(), timeout=1)
564570
if status == 0:
565571
exit(0)
566572
else:
567573
raise CalledProcessError(status, args)
568574
finally:
569-
# If logcat never started, then something has gone badly wrong, so the
570-
# user probably wants to see the Gradle output even in non-verbose mode.
571-
if hidden_output and not logcat_started:
572-
sys.stdout.write("".join(hidden_output))
573-
574575
# Gradle does not stop the tests when interrupted.
575576
if context.connected:
576577
stop_app(context.connected)
@@ -600,6 +601,12 @@ async def run_testbed(context):
600601
except* MySystemExit as e:
601602
raise SystemExit(*e.exceptions[0].args) from None
602603
except* CalledProcessError as e:
604+
# If Python produced no output, then the user probably wants to see the
605+
# verbose output to explain why the test failed.
606+
if not python_started:
607+
for stream, line in hidden_output:
608+
stream.write(line)
609+
603610
# Extract it from the ExceptionGroup so it can be handled by `main`.
604611
raise e.exceptions[0]
605612

‎Doc/c-api/arg.rst

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C:
113113
``z`` (:class:`str` or ``None``) [const char \*]
114114
Like ``s``, but the Python object may also be ``None``, in which case the C
115115
pointer is set to ``NULL``.
116-
It is the same as ``s?`` with the C pointer was initialized to ``NULL``.
117116

118117
``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer]
119118
Like ``s*``, but the Python object may also be ``None``, in which case the
120119
``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``.
121-
It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer`
122-
structure was initialized to ``NULL``.
123120

124121
``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`]
125122
Like ``s#``, but the Python object may also be ``None``, in which case the C
126123
pointer is set to ``NULL``.
127-
It is the same as ``s#?`` with the C pointer was initialized to ``NULL``.
128124

129125
``y`` (read-only :term:`bytes-like object`) [const char \*]
130126
This format converts a bytes-like object to a C pointer to a
@@ -394,17 +390,6 @@ Other objects
394390
Non-tuple sequences are deprecated if *items* contains format units
395391
which store a borrowed buffer or a borrowed reference.
396392

397-
``unit?`` (anything or ``None``) [*matching-variable(s)*]
398-
``?`` modifies the behavior of the preceding format unit.
399-
The C variable(s) corresponding to that parameter should be initialized
400-
to their default value --- when the argument is ``None``,
401-
:c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding
402-
C variable(s).
403-
If the argument is not ``None``, it is parsed according to the specified
404-
format unit.
405-
406-
.. versionadded:: 3.14
407-
408393
A few other characters have a meaning in a format string. These may not occur
409394
inside nested parentheses. They are:
410395

‎Doc/c-api/init.rst

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2306,6 +2306,12 @@ is resumed, and its locks reacquired. This means the critical section API
23062306
provides weaker guarantees than traditional locks -- they are useful because
23072307
their behavior is similar to the :term:`GIL`.
23082308
2309+
Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also
2310+
available. Use these variants to start a critical section in a situation where
2311+
there is no :c:type:`PyObject` -- for example, when working with a C type that
2312+
does not extend or wrap :c:type:`PyObject` but still needs to call into the C
2313+
API in a manner that might lead to deadlocks.
2314+
23092315
The functions and structs used by the macros are exposed for cases
23102316
where C macros are not available. They should only be used as in the
23112317
given macro expansions. Note that the sizes and contents of the structures may
@@ -2351,6 +2357,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`.
23512357
23522358
.. versionadded:: 3.13
23532359
2360+
.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m)
2361+
2362+
Locks the mutex *m* and begins a critical section.
2363+
2364+
In the free-threaded build, this macro expands to::
2365+
2366+
{
2367+
PyCriticalSection _py_cs;
2368+
PyCriticalSection_BeginMutex(&_py_cs, m)
2369+
2370+
Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for
2371+
the argument of the macro - it must be a :c:type:`PyMutex` pointer.
2372+
2373+
On the default build, this macro expands to ``{``.
2374+
2375+
.. versionadded:: next
2376+
23542377
.. c:macro:: Py_END_CRITICAL_SECTION()
23552378
23562379
Ends the critical section and releases the per-object lock.
@@ -2380,6 +2403,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`.
23802403
23812404
.. versionadded:: 3.13
23822405
2406+
.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2)
2407+
2408+
Locks the mutexes *m1* and *m2* and begins a critical section.
2409+
2410+
In the free-threaded build, this macro expands to::
2411+
2412+
{
2413+
PyCriticalSection2 _py_cs2;
2414+
PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2)
2415+
2416+
Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for
2417+
the arguments of the macro - they must be :c:type:`PyMutex` pointers.
2418+
2419+
On the default build, this macro expands to ``{``.
2420+
2421+
.. versionadded:: next
2422+
23832423
.. c:macro:: Py_END_CRITICAL_SECTION2()
23842424
23852425
Ends the critical section and releases the per-object locks.

‎Doc/c-api/perfmaps.rst

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
Support for Perf Maps
66
----------------------
77

8-
On supported platforms (as of this writing, only Linux), the runtime can take
8+
On supported platforms (Linux and macOS), the runtime can take
99
advantage of *perf map files* to make Python functions visible to an external
10-
profiling tool (such as `perf <https://perf.wiki.kernel.org/index.php/Main_Page>`_).
11-
A running process may create a file in the ``/tmp`` directory, which contains entries
12-
that can map a section of executable code to a name. This interface is described in the
10+
profiling tool (such as `perf <https://perf.wiki.kernel.org/index.php/Main_Page>`_ or
11+
`samply <https://github.com/mstange/samply/>`_). A running process may create a
12+
file in the ``/tmp`` directory, which contains entries that can map a section
13+
of executable code to a name. This interface is described in the
1314
`documentation of the Linux Perf tool <https://git.kernel.org/pub/scm/linux/
1415
kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt>`_.
1516

‎Doc/glossary.rst

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ Glossary
462462
core and with user code.
463463

464464
f-string
465-
String literals prefixed with ``'f'`` or ``'F'`` are commonly called
465+
String literals prefixed with ``f`` or ``F`` are commonly called
466466
"f-strings" which is short for
467467
:ref:`formatted string literals <f-strings>`. See also :pep:`498`.
468468

@@ -1322,6 +1322,11 @@ Glossary
13221322

13231323
See also :term:`borrowed reference`.
13241324

1325+
t-string
1326+
String literals prefixed with ``t`` or ``T`` are commonly called
1327+
"t-strings" which is short for
1328+
:ref:`template string literals <t-strings>`.
1329+
13251330
text encoding
13261331
A string in Python is a sequence of Unicode code points (in range
13271332
``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be

‎Doc/howto/perf_profiling.rst

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,35 @@
22

33
.. _perf_profiling:
44

5-
==============================================
6-
Python support for the Linux ``perf`` profiler
7-
==============================================
5+
========================================================
6+
Python support for the ``perf map`` compatible profilers
7+
========================================================
88

99
:author: Pablo Galindo
1010

11-
`The Linux perf profiler <https://perf.wiki.kernel.org>`_
12-
is a very powerful tool that allows you to profile and obtain
13-
information about the performance of your application.
14-
``perf`` also has a very vibrant ecosystem of tools
15-
that aid with the analysis of the data that it produces.
11+
`The Linux perf profiler <https://perf.wiki.kernel.org>`_ and
12+
`samply <https://github.com/mstange/samply>`_ are powerful tools that allow you to
13+
profile and obtain information about the performance of your application.
14+
Both tools have vibrant ecosystems that aid with the analysis of the data they produce.
1615

17-
The main problem with using the ``perf`` profiler with Python applications is that
18-
``perf`` only gets information about native symbols, that is, the names of
16+
The main problem with using these profilers with Python applications is that
17+
they only get information about native symbols, that is, the names of
1918
functions and procedures written in C. This means that the names and file names
20-
of Python functions in your code will not appear in the output of ``perf``.
19+
of Python functions in your code will not appear in the profiler output.
2120

2221
Since Python 3.12, the interpreter can run in a special mode that allows Python
23-
functions to appear in the output of the ``perf`` profiler. When this mode is
22+
functions to appear in the output of compatible profilers. When this mode is
2423
enabled, the interpreter will interpose a small piece of code compiled on the
25-
fly before the execution of every Python function and it will teach ``perf`` the
24+
fly before the execution of every Python function and it will teach the profiler the
2625
relationship between this piece of code and the associated Python function using
2726
:doc:`perf map files <../c-api/perfmaps>`.
2827

2928
.. note::
3029

31-
Support for the ``perf`` profiler is currently only available for Linux on
32-
select architectures. Check the output of the ``configure`` build step or
30+
Support for profiling is available on Linux and macOS on select architectures.
31+
Perf is available on Linux, while samply can be used on both Linux and macOS.
32+
samply support on macOS is available starting from Python 3.15.
33+
Check the output of the ``configure`` build step or
3334
check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE``
3435
to see if your system is supported.
3536

@@ -148,6 +149,31 @@ Instead, if we run the same experiment with ``perf`` support enabled we get:
148149
149150
150151
152+
Using the samply profiler
153+
-------------------------
154+
155+
samply is a modern profiler that can be used as an alternative to perf.
156+
It uses the same perf map files that Python generates, making it compatible
157+
with Python's profiling support. samply is particularly useful on macOS
158+
where perf is not available.
159+
160+
To use samply with Python, first install it following the instructions at
161+
https://github.com/mstange/samply, then run::
162+
163+
$ samply record PYTHONPERFSUPPORT=1 python my_script.py
164+
165+
This will open a web interface where you can analyze the profiling data
166+
interactively. The advantage of samply is that it provides a modern
167+
web-based interface for analyzing profiling data and works on both Linux
168+
and macOS.
169+
170+
On macOS, samply support requires Python 3.15 or later. Also on macOS, samply
171+
can't profile signed Python executables due to restrictions by macOS. You can
172+
profile with Python binaries that you've compiled yourself, or which are
173+
unsigned or locally-signed (such as anything installed by Homebrew). In
174+
order to attach to running processes on macOS, run ``samply setup`` once (and
175+
every time samply is updated) to self-sign the samply binary.
176+
151177
How to enable ``perf`` profiling support
152178
----------------------------------------
153179

‎Doc/library/annotationlib.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ code execution even with no access to any globals or builtins. For example:
511511
512512
>>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass
513513
...
514-
>>> annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE)
514+
>>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING)
515515
Hello world
516516
{'x': 'None'}
517517

0 commit comments

Comments
 (0)