Skip to content

Commit 59df92d

Browse files
youknowoneyouknowone
authored
Merge pull request #4981 from DimitrisJim/update_subprocess
Update subprocess to CPython 3.11
2 parents 9ba6027 + 283a804 commit 59df92d

File tree

3 files changed

+201
-91
lines changed

3 files changed

+201
-91
lines changed

‎Lib/subprocess.py

Lines changed: 92 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import builtins
4444
import errno
4545
import io
46+
import locale
4647
import os
4748
import time
4849
import signal
@@ -65,16 +66,19 @@
6566
# NOTE: We intentionally exclude list2cmdline as it is
6667
# considered an internal implementation detail. issue10838.
6768

69+
# use presence of msvcrt to detect Windows-like platforms (see bpo-8110)
6870
try:
6971
import msvcrt
70-
import _winapi
71-
_mswindows = True
7272
except ModuleNotFoundError:
7373
_mswindows = False
74-
import _posixsubprocess
75-
import select
76-
import selectors
7774
else:
75+
_mswindows = True
76+
77+
# wasm32-emscripten and wasm32-wasi do not support processes
78+
_can_fork_exec = sys.platform not in {"emscripten", "wasi"}
79+
80+
if _mswindows:
81+
import _winapi
7882
from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
7983
STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
8084
STD_ERROR_HANDLE, SW_HIDE,
@@ -95,6 +99,24 @@
9599
"NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
96100
"CREATE_NO_WINDOW", "DETACHED_PROCESS",
97101
"CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
102+
else:
103+
if _can_fork_exec:
104+
from _posixsubprocess import fork_exec as _fork_exec
105+
# used in methods that are called by __del__
106+
_waitpid = os.waitpid
107+
_waitstatus_to_exitcode = os.waitstatus_to_exitcode
108+
_WIFSTOPPED = os.WIFSTOPPED
109+
_WSTOPSIG = os.WSTOPSIG
110+
_WNOHANG = os.WNOHANG
111+
else:
112+
_fork_exec = None
113+
_waitpid = None
114+
_waitstatus_to_exitcode = None
115+
_WIFSTOPPED = None
116+
_WSTOPSIG = None
117+
_WNOHANG = None
118+
import select
119+
import selectors
98120

99121

100122
# Exception classes used by this module.
@@ -207,8 +229,7 @@ def Detach(self):
207229
def __repr__(self):
208230
return "%s(%d)" % (self.__class__.__name__, int(self))
209231

210-
# XXX: RustPython; OSError('The handle is invalid. (os error 6)')
211-
# __del__ = Close
232+
__del__ = Close
212233
else:
213234
# When select or poll has indicated that the file is writable,
214235
# we can write up to _PIPE_BUF bytes without risk of blocking.
@@ -303,12 +324,14 @@ def _args_from_interpreter_flags():
303324
args.append('-E')
304325
if sys.flags.no_user_site:
305326
args.append('-s')
327+
if sys.flags.safe_path:
328+
args.append('-P')
306329

307330
# -W options
308331
warnopts = sys.warnoptions[:]
309-
bytes_warning = sys.flags.bytes_warning
310332
xoptions = getattr(sys, '_xoptions', {})
311-
dev_mode = ('dev' in xoptions)
333+
bytes_warning = sys.flags.bytes_warning
334+
dev_mode = sys.flags.dev_mode
312335

313336
if bytes_warning > 1:
314337
warnopts.remove("error::BytesWarning")
@@ -335,6 +358,26 @@ def _args_from_interpreter_flags():
335358
return args
336359

337360

361+
def _text_encoding():
362+
# Return default text encoding and emit EncodingWarning if
363+
# sys.flags.warn_default_encoding is true.
364+
if sys.flags.warn_default_encoding:
365+
f = sys._getframe()
366+
filename = f.f_code.co_filename
367+
stacklevel = 2
368+
while f := f.f_back:
369+
if f.f_code.co_filename != filename:
370+
break
371+
stacklevel += 1
372+
warnings.warn("'encoding' argument not specified.",
373+
EncodingWarning, stacklevel)
374+
375+
if sys.flags.utf8_mode:
376+
return "utf-8"
377+
else:
378+
return locale.getencoding()
379+
380+
338381
def call(*popenargs, timeout=None, **kwargs):
339382
"""Run command with arguments. Wait for command to complete or
340383
timeout, then return the returncode attribute.
@@ -406,13 +449,15 @@ def check_output(*popenargs, timeout=None, **kwargs):
406449
decoded according to locale encoding, or by "encoding" if set. Text mode
407450
is triggered by setting any of text, encoding, errors or universal_newlines.
408451
"""
409-
if 'stdout' in kwargs:
410-
raise ValueError('stdout argument not allowed, it will be overridden.')
452+
for kw in ('stdout', 'check'):
453+
if kw in kwargs:
454+
raise ValueError(f'{kw} argument not allowed, it will be overridden.')
411455

412456
if 'input' in kwargs and kwargs['input'] is None:
413457
# Explicitly passing input=None was previously equivalent to passing an
414458
# empty string. That is maintained here for backwards compatibility.
415-
if kwargs.get('universal_newlines') or kwargs.get('text'):
459+
if kwargs.get('universal_newlines') or kwargs.get('text') or kwargs.get('encoding') \
460+
or kwargs.get('errors'):
416461
empty = ''
417462
else:
418463
empty = b''
@@ -464,7 +509,8 @@ def run(*popenargs,
464509
465510
The returned instance will have attributes args, returncode, stdout and
466511
stderr. By default, stdout and stderr are not captured, and those attributes
467-
will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
512+
will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them,
513+
or pass capture_output=True to capture both.
468514
469515
If check is True and the exit code was non-zero, it raises a
470516
CalledProcessError. The CalledProcessError object will have the return code
@@ -600,7 +646,7 @@ def list2cmdline(seq):
600646
# Various tools for executing commands and looking at their output and status.
601647
#
602648

603-
def getstatusoutput(cmd):
649+
def getstatusoutput(cmd, *, encoding=None, errors=None):
604650
"""Return (exitcode, output) of executing cmd in a shell.
605651
606652
Execute the string 'cmd' in a shell with 'check_output' and
@@ -622,7 +668,8 @@ def getstatusoutput(cmd):
622668
(-15, '')
623669
"""
624670
try:
625-
data = check_output(cmd, shell=True, text=True, stderr=STDOUT)
671+
data = check_output(cmd, shell=True, text=True, stderr=STDOUT,
672+
encoding=encoding, errors=errors)
626673
exitcode = 0
627674
except CalledProcessError as ex:
628675
data = ex.output
@@ -631,7 +678,7 @@ def getstatusoutput(cmd):
631678
data = data[:-1]
632679
return exitcode, data
633680

634-
def getoutput(cmd):
681+
def getoutput(cmd, *, encoding=None, errors=None):
635682
"""Return output (stdout or stderr) of executing cmd in a shell.
636683
637684
Like getstatusoutput(), except the exit status is ignored and the return
@@ -641,7 +688,8 @@ def getoutput(cmd):
641688
>>> subprocess.getoutput('ls /bin/ls')
642689
'/bin/ls'
643690
"""
644-
return getstatusoutput(cmd)[1]
691+
return getstatusoutput(cmd, encoding=encoding, errors=errors)[1]
692+
645693

646694

647695
def _use_posix_spawn():
@@ -736,6 +784,8 @@ class Popen:
736784
737785
start_new_session (POSIX only)
738786
787+
process_group (POSIX only)
788+
739789
group (POSIX only)
740790
741791
extra_groups (POSIX only)
@@ -761,8 +811,14 @@ def __init__(self, args, bufsize=-1, executable=None,
761811
startupinfo=None, creationflags=0,
762812
restore_signals=True, start_new_session=False,
763813
pass_fds=(), *, user=None, group=None, extra_groups=None,
764-
encoding=None, errors=None, text=None, umask=-1, pipesize=-1):
814+
encoding=None, errors=None, text=None, umask=-1, pipesize=-1,
815+
process_group=None):
765816
"""Create new Popen instance."""
817+
if not _can_fork_exec:
818+
raise OSError(
819+
errno.ENOTSUP, f"{sys.platform} does not support processes."
820+
)
821+
766822
_cleanup()
767823
# Held while anything is calling waitpid before returncode has been
768824
# updated to prevent clobbering returncode if wait() or poll() are
@@ -848,15 +904,8 @@ def __init__(self, args, bufsize=-1, executable=None,
848904
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
849905

850906
self.text_mode = encoding or errors or text or universal_newlines
851-
852-
# PEP 597: We suppress the EncodingWarning in subprocess module
853-
# for now (at Python 3.10), because we focus on files for now.
854-
# This will be changed to encoding = io.text_encoding(encoding)
855-
# in the future.
856907
if self.text_mode and encoding is None:
857-
# TODO: RUSTPYTHON; encoding `locale` is not supported yet
858-
pass
859-
# self.encoding = encoding = "locale"
908+
self.encoding = encoding = _text_encoding()
860909

861910
# How long to resume waiting on a child after the first ^C.
862911
# There is no right value for this. The purpose is to be polite
@@ -874,6 +923,9 @@ def __init__(self, args, bufsize=-1, executable=None,
874923
else:
875924
line_buffering = False
876925

926+
if process_group is None:
927+
process_group = -1 # The internal APIs are int-only
928+
877929
gid = None
878930
if group is not None:
879931
if not hasattr(os, 'setregid'):
@@ -977,7 +1029,7 @@ def __init__(self, args, bufsize=-1, executable=None,
9771029
errread, errwrite,
9781030
restore_signals,
9791031
gid, gids, uid, umask,
980-
start_new_session)
1032+
start_new_session, process_group)
9811033
except:
9821034
# Cleanup if the child failed starting.
9831035
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
@@ -1285,11 +1337,7 @@ def _get_handles(self, stdin, stdout, stderr):
12851337
else:
12861338
# Assuming file-like object
12871339
p2cread = msvcrt.get_osfhandle(stdin.fileno())
1288-
# XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable
1289-
# pipe handles is necessary for us, but not CPython
1290-
old = p2cread
12911340
p2cread = self._make_inheritable(p2cread)
1292-
if stdin == PIPE: _winapi.CloseHandle(old)
12931341

12941342
if stdout is None:
12951343
c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
@@ -1307,11 +1355,7 @@ def _get_handles(self, stdin, stdout, stderr):
13071355
else:
13081356
# Assuming file-like object
13091357
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1310-
# XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable
1311-
# pipe handles is necessary for us, but not CPython
1312-
old = c2pwrite
13131358
c2pwrite = self._make_inheritable(c2pwrite)
1314-
if stdout == PIPE: _winapi.CloseHandle(old)
13151359

13161360
if stderr is None:
13171361
errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
@@ -1331,11 +1375,7 @@ def _get_handles(self, stdin, stdout, stderr):
13311375
else:
13321376
# Assuming file-like object
13331377
errwrite = msvcrt.get_osfhandle(stderr.fileno())
1334-
# XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable
1335-
# pipe handles is necessary for us, but not CPython
1336-
old = errwrite
13371378
errwrite = self._make_inheritable(errwrite)
1338-
if stderr == PIPE: _winapi.CloseHandle(old)
13391379

13401380
return (p2cread, p2cwrite,
13411381
c2pread, c2pwrite,
@@ -1373,7 +1413,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
13731413
unused_restore_signals,
13741414
unused_gid, unused_gids, unused_uid,
13751415
unused_umask,
1376-
unused_start_new_session):
1416+
unused_start_new_session, unused_process_group):
13771417
"""Execute program (MS Windows version)"""
13781418

13791419
assert not pass_fds, "pass_fds not supported on Windows."
@@ -1705,7 +1745,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
17051745
errread, errwrite,
17061746
restore_signals,
17071747
gid, gids, uid, umask,
1708-
start_new_session):
1748+
start_new_session, process_group):
17091749
"""Execute program (POSIX version)"""
17101750

17111751
if isinstance(args, (str, bytes)):
@@ -1741,6 +1781,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
17411781
and (c2pwrite == -1 or c2pwrite > 2)
17421782
and (errwrite == -1 or errwrite > 2)
17431783
and not start_new_session
1784+
and process_group == -1
17441785
and gid is None
17451786
and gids is None
17461787
and uid is None
@@ -1790,16 +1831,16 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
17901831
for dir in os.get_exec_path(env))
17911832
fds_to_keep = set(pass_fds)
17921833
fds_to_keep.add(errpipe_write)
1793-
self.pid = _posixsubprocess.fork_exec(
1834+
self.pid = _fork_exec(
17941835
args, executable_list,
17951836
close_fds, tuple(sorted(map(int, fds_to_keep))),
17961837
cwd, env_list,
17971838
p2cread, p2cwrite, c2pread, c2pwrite,
17981839
errread, errwrite,
17991840
errpipe_read, errpipe_write,
18001841
restore_signals, start_new_session,
1801-
gid, gids, uid, umask,
1802-
preexec_fn)
1842+
process_group, gid, gids, uid, umask,
1843+
preexec_fn, _USE_VFORK)
18031844
self._child_created = True
18041845
finally:
18051846
# be sure the FD is closed no matter what
@@ -1862,19 +1903,19 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
18621903

18631904

18641905
def _handle_exitstatus(self, sts,
1865-
waitstatus_to_exitcode=os.waitstatus_to_exitcode,
1866-
_WIFSTOPPED=os.WIFSTOPPED,
1867-
_WSTOPSIG=os.WSTOPSIG):
1906+
_waitstatus_to_exitcode=_waitstatus_to_exitcode,
1907+
_WIFSTOPPED=_WIFSTOPPED,
1908+
_WSTOPSIG=_WSTOPSIG):
18681909
"""All callers to this function MUST hold self._waitpid_lock."""
18691910
# This method is called (indirectly) by __del__, so it cannot
18701911
# refer to anything outside of its local scope.
18711912
if _WIFSTOPPED(sts):
18721913
self.returncode = -_WSTOPSIG(sts)
18731914
else:
1874-
self.returncode = waitstatus_to_exitcode(sts)
1915+
self.returncode = _waitstatus_to_exitcode(sts)
18751916

1876-
def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
1877-
_WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
1917+
def _internal_poll(self, _deadstate=None, _waitpid=_waitpid,
1918+
_WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD):
18781919
"""Check if child process has terminated. Returns returncode
18791920
attribute.
18801921
@@ -2105,7 +2146,7 @@ def send_signal(self, sig):
21052146
try:
21062147
os.kill(self.pid, sig)
21072148
except ProcessLookupError:
2108-
# Supress the race condition error; bpo-40550.
2149+
# Suppress the race condition error; bpo-40550.
21092150
pass
21102151

21112152
def terminate(self):

0 commit comments

Comments
 (0)