Skip to content

Update subprocess to CPython 3.11 #4981

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 20, 2023
Merged

Update subprocess to CPython 3.11 #4981

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2b43d48
Add additional arguments to fork_exec.
DimitrisJim May 19, 2023
7366a41
Update subprocess from CPython 3.11
DimitrisJim May 19, 2023
283a804
Mark failing tests.
DimitrisJim May 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 92 additions & 51 deletions Lib/subprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import builtins
import errno
import io
import locale
import os
import time
import signal
Expand All @@ -65,16 +66,19 @@
# NOTE: We intentionally exclude list2cmdline as it is
# considered an internal implementation detail. issue10838.

# use presence of msvcrt to detect Windows-like platforms (see bpo-8110)
try:
import msvcrt
import _winapi
_mswindows = True
except ModuleNotFoundError:
_mswindows = False
import _posixsubprocess
import select
import selectors
else:
_mswindows = True

# wasm32-emscripten and wasm32-wasi do not support processes
_can_fork_exec = sys.platform not in {"emscripten", "wasi"}

if _mswindows:
import _winapi
from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
STD_ERROR_HANDLE, SW_HIDE,
Expand All @@ -95,6 +99,24 @@
"NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
"CREATE_NO_WINDOW", "DETACHED_PROCESS",
"CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
else:
if _can_fork_exec:
from _posixsubprocess import fork_exec as _fork_exec
# used in methods that are called by __del__
_waitpid = os.waitpid
_waitstatus_to_exitcode = os.waitstatus_to_exitcode
_WIFSTOPPED = os.WIFSTOPPED
_WSTOPSIG = os.WSTOPSIG
_WNOHANG = os.WNOHANG
else:
_fork_exec = None
_waitpid = None
_waitstatus_to_exitcode = None
_WIFSTOPPED = None
_WSTOPSIG = None
_WNOHANG = None
import select
import selectors


# Exception classes used by this module.
Expand Down Expand Up @@ -207,8 +229,7 @@ def Detach(self):
def __repr__(self):
return "%s(%d)" % (self.__class__.__name__, int(self))

# XXX: RustPython; OSError('The handle is invalid. (os error 6)')
# __del__ = Close
__del__ = Close
else:
# When select or poll has indicated that the file is writable,
# we can write up to _PIPE_BUF bytes without risk of blocking.
Expand Down Expand Up @@ -303,12 +324,14 @@ def _args_from_interpreter_flags():
args.append('-E')
if sys.flags.no_user_site:
args.append('-s')
if sys.flags.safe_path:
args.append('-P')

# -W options
warnopts = sys.warnoptions[:]
bytes_warning = sys.flags.bytes_warning
xoptions = getattr(sys, '_xoptions', {})
dev_mode = ('dev' in xoptions)
bytes_warning = sys.flags.bytes_warning
dev_mode = sys.flags.dev_mode

if bytes_warning > 1:
warnopts.remove("error::BytesWarning")
Expand All @@ -335,6 +358,26 @@ def _args_from_interpreter_flags():
return args


def _text_encoding():
# Return default text encoding and emit EncodingWarning if
# sys.flags.warn_default_encoding is true.
if sys.flags.warn_default_encoding:
f = sys._getframe()
filename = f.f_code.co_filename
stacklevel = 2
while f := f.f_back:
if f.f_code.co_filename != filename:
break
stacklevel += 1
warnings.warn("'encoding' argument not specified.",
EncodingWarning, stacklevel)

if sys.flags.utf8_mode:
return "utf-8"
else:
return locale.getencoding()


def call(*popenargs, timeout=None, **kwargs):
"""Run command with arguments. Wait for command to complete or
timeout, then return the returncode attribute.
Expand Down Expand Up @@ -406,13 +449,15 @@ def check_output(*popenargs, timeout=None, **kwargs):
decoded according to locale encoding, or by "encoding" if set. Text mode
is triggered by setting any of text, encoding, errors or universal_newlines.
"""
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
for kw in ('stdout', 'check'):
if kw in kwargs:
raise ValueError(f'{kw} argument not allowed, it will be overridden.')

if 'input' in kwargs and kwargs['input'] is None:
# Explicitly passing input=None was previously equivalent to passing an
# empty string. That is maintained here for backwards compatibility.
if kwargs.get('universal_newlines') or kwargs.get('text'):
if kwargs.get('universal_newlines') or kwargs.get('text') or kwargs.get('encoding') \
or kwargs.get('errors'):
empty = ''
else:
empty = b''
Expand Down Expand Up @@ -464,7 +509,8 @@ def run(*popenargs,

The returned instance will have attributes args, returncode, stdout and
stderr. By default, stdout and stderr are not captured, and those attributes
will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them,
or pass capture_output=True to capture both.

If check is True and the exit code was non-zero, it raises a
CalledProcessError. The CalledProcessError object will have the return code
Expand Down Expand Up @@ -600,7 +646,7 @@ def list2cmdline(seq):
# Various tools for executing commands and looking at their output and status.
#

def getstatusoutput(cmd):
def getstatusoutput(cmd, *, encoding=None, errors=None):
"""Return (exitcode, output) of executing cmd in a shell.

Execute the string 'cmd' in a shell with 'check_output' and
Expand All @@ -622,7 +668,8 @@ def getstatusoutput(cmd):
(-15, '')
"""
try:
data = check_output(cmd, shell=True, text=True, stderr=STDOUT)
data = check_output(cmd, shell=True, text=True, stderr=STDOUT,
encoding=encoding, errors=errors)
exitcode = 0
except CalledProcessError as ex:
data = ex.output
Expand All @@ -631,7 +678,7 @@ def getstatusoutput(cmd):
data = data[:-1]
return exitcode, data

def getoutput(cmd):
def getoutput(cmd, *, encoding=None, errors=None):
"""Return output (stdout or stderr) of executing cmd in a shell.

Like getstatusoutput(), except the exit status is ignored and the return
Expand All @@ -641,7 +688,8 @@ def getoutput(cmd):
>>> subprocess.getoutput('ls /bin/ls')
'/bin/ls'
"""
return getstatusoutput(cmd)[1]
return getstatusoutput(cmd, encoding=encoding, errors=errors)[1]



def _use_posix_spawn():
Expand Down Expand Up @@ -736,6 +784,8 @@ class Popen:

start_new_session (POSIX only)

process_group (POSIX only)

group (POSIX only)

extra_groups (POSIX only)
Expand All @@ -761,8 +811,14 @@ def __init__(self, args, bufsize=-1, executable=None,
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False,
pass_fds=(), *, user=None, group=None, extra_groups=None,
encoding=None, errors=None, text=None, umask=-1, pipesize=-1):
encoding=None, errors=None, text=None, umask=-1, pipesize=-1,
process_group=None):
"""Create new Popen instance."""
if not _can_fork_exec:
raise OSError(
errno.ENOTSUP, f"{sys.platform} does not support processes."
)

_cleanup()
# Held while anything is calling waitpid before returncode has been
# updated to prevent clobbering returncode if wait() or poll() are
Expand Down Expand Up @@ -848,15 +904,8 @@ def __init__(self, args, bufsize=-1, executable=None,
errread = msvcrt.open_osfhandle(errread.Detach(), 0)

self.text_mode = encoding or errors or text or universal_newlines

# PEP 597: We suppress the EncodingWarning in subprocess module
# for now (at Python 3.10), because we focus on files for now.
# This will be changed to encoding = io.text_encoding(encoding)
# in the future.
if self.text_mode and encoding is None:
# TODO: RUSTPYTHON; encoding `locale` is not supported yet
pass
# self.encoding = encoding = "locale"
self.encoding = encoding = _text_encoding()

# How long to resume waiting on a child after the first ^C.
# There is no right value for this. The purpose is to be polite
Expand All @@ -874,6 +923,9 @@ def __init__(self, args, bufsize=-1, executable=None,
else:
line_buffering = False

if process_group is None:
process_group = -1 # The internal APIs are int-only

gid = None
if group is not None:
if not hasattr(os, 'setregid'):
Expand Down Expand Up @@ -977,7 +1029,7 @@ def __init__(self, args, bufsize=-1, executable=None,
errread, errwrite,
restore_signals,
gid, gids, uid, umask,
start_new_session)
start_new_session, process_group)
except:
# Cleanup if the child failed starting.
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
Expand Down Expand Up @@ -1285,11 +1337,7 @@ def _get_handles(self, stdin, stdout, stderr):
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
# XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable
# pipe handles is necessary for us, but not CPython
old = p2cread
p2cread = self._make_inheritable(p2cread)
if stdin == PIPE: _winapi.CloseHandle(old)

if stdout is None:
c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
Expand All @@ -1307,11 +1355,7 @@ def _get_handles(self, stdin, stdout, stderr):
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
# XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable
# pipe handles is necessary for us, but not CPython
old = c2pwrite
c2pwrite = self._make_inheritable(c2pwrite)
if stdout == PIPE: _winapi.CloseHandle(old)

if stderr is None:
errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
Expand All @@ -1331,11 +1375,7 @@ def _get_handles(self, stdin, stdout, stderr):
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
# XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable
# pipe handles is necessary for us, but not CPython
old = errwrite
errwrite = self._make_inheritable(errwrite)
if stderr == PIPE: _winapi.CloseHandle(old)

return (p2cread, p2cwrite,
c2pread, c2pwrite,
Expand Down Expand Up @@ -1373,7 +1413,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
unused_restore_signals,
unused_gid, unused_gids, unused_uid,
unused_umask,
unused_start_new_session):
unused_start_new_session, unused_process_group):
"""Execute program (MS Windows version)"""

assert not pass_fds, "pass_fds not supported on Windows."
Expand Down Expand Up @@ -1705,7 +1745,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
errread, errwrite,
restore_signals,
gid, gids, uid, umask,
start_new_session):
start_new_session, process_group):
"""Execute program (POSIX version)"""

if isinstance(args, (str, bytes)):
Expand Down Expand Up @@ -1741,6 +1781,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
and (c2pwrite == -1 or c2pwrite > 2)
and (errwrite == -1 or errwrite > 2)
and not start_new_session
and process_group == -1
and gid is None
and gids is None
and uid is None
Expand Down Expand Up @@ -1790,16 +1831,16 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
self.pid = _fork_exec(
args, executable_list,
close_fds, tuple(sorted(map(int, fds_to_keep))),
cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session,
gid, gids, uid, umask,
preexec_fn)
process_group, gid, gids, uid, umask,
preexec_fn, _USE_VFORK)
self._child_created = True
finally:
# be sure the FD is closed no matter what
Expand Down Expand Up @@ -1862,19 +1903,19 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,


def _handle_exitstatus(self, sts,
waitstatus_to_exitcode=os.waitstatus_to_exitcode,
_WIFSTOPPED=os.WIFSTOPPED,
_WSTOPSIG=os.WSTOPSIG):
_waitstatus_to_exitcode=_waitstatus_to_exitcode,
_WIFSTOPPED=_WIFSTOPPED,
_WSTOPSIG=_WSTOPSIG):
"""All callers to this function MUST hold self._waitpid_lock."""
# This method is called (indirectly) by __del__, so it cannot
# refer to anything outside of its local scope.
if _WIFSTOPPED(sts):
self.returncode = -_WSTOPSIG(sts)
else:
self.returncode = waitstatus_to_exitcode(sts)
self.returncode = _waitstatus_to_exitcode(sts)

def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
_WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
def _internal_poll(self, _deadstate=None, _waitpid=_waitpid,
_WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD):
"""Check if child process has terminated. Returns returncode
attribute.

Expand Down Expand Up @@ -2105,7 +2146,7 @@ def send_signal(self, sig):
try:
os.kill(self.pid, sig)
except ProcessLookupError:
# Supress the race condition error; bpo-40550.
# Suppress the race condition error; bpo-40550.
pass

def terminate(self):
Expand Down
Loading