From 02aa9f4395617beb33fdbd3b5fa4bf1486c2a246 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 11:05:17 +0900 Subject: [PATCH 1/8] Emit EncodingWarning locale.getpreferredencoding() and subprocess.Popen() emit EncodingWarning --- Lib/locale.py | 11 +++++++++++ Lib/subprocess.py | 8 ++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py index 170e5eea45b8ca..a2825350a28f9c 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -655,6 +655,11 @@ def getencoding(): except NameError: def getpreferredencoding(do_setlocale=True): """Return the charset that the user is likely using.""" + if sys.flags.warn_default_encoding: + import warnings + warnings.warn( + "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.", + EncodingWarning, stacklevel + 1) if sys.flags.utf8_mode: return 'utf-8' return getencoding() @@ -663,6 +668,12 @@ def getpreferredencoding(do_setlocale=True): def getpreferredencoding(do_setlocale=True): """Return the charset that the user is likely using, according to the system configuration.""" + + if sys.flags.warn_default_encoding: + import warnings + warnings.warn( + "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.", + EncodingWarning, stacklevel + 1) if sys.flags.utf8_mode: return 'utf-8' diff --git a/Lib/subprocess.py b/Lib/subprocess.py index a5fa152715c145..2cab4edf42be27 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -859,12 +859,8 @@ def __init__(self, args, bufsize=-1, executable=None, self.text_mode = encoding or errors or text or universal_newlines - # PEP 597: We suppress the EncodingWarning in subprocess module - # for now (at Python 3.10), because we focus on files for now. - # This will be changed to encoding = io.text_encoding(encoding) - # in the future. - if self.text_mode and encoding is None: - self.encoding = encoding = "locale" + if self.text_mode: + self.encoding = encoding = io.text_encoding(encoding) # How long to resume waiting on a child after the first ^C. # There is no right value for this. The purpose is to be polite From 3a6324b2f0cf7a5b54893c8458221f8e993cbea9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 11:13:01 +0900 Subject: [PATCH 2/8] add comment --- Lib/subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 2cab4edf42be27..4638485975f1a3 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -858,8 +858,8 @@ def __init__(self, args, bufsize=-1, executable=None, errread = msvcrt.open_osfhandle(errread.Detach(), 0) self.text_mode = encoding or errors or text or universal_newlines - if self.text_mode: + # Emit EncodingWarning when text mode but *encoding* is not specified. self.encoding = encoding = io.text_encoding(encoding) # How long to resume waiting on a child after the first ^C. From 903029b46fe3c6ad6463e862e6646e9c0cb6d75c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 11:22:02 +0900 Subject: [PATCH 3/8] fixup --- Lib/locale.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py index a2825350a28f9c..25eb75ac65a32b 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -659,7 +659,7 @@ def getpreferredencoding(do_setlocale=True): import warnings warnings.warn( "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.", - EncodingWarning, stacklevel + 1) + EncodingWarning, 2) if sys.flags.utf8_mode: return 'utf-8' return getencoding() @@ -673,7 +673,7 @@ def getpreferredencoding(do_setlocale=True): import warnings warnings.warn( "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.", - EncodingWarning, stacklevel + 1) + EncodingWarning, 2) if sys.flags.utf8_mode: return 'utf-8' From 91637230d0fa6c73419211ed9e2febc81caf6c06 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 13:08:04 +0900 Subject: [PATCH 4/8] Better EncodingWarning from subprocess --- Lib/subprocess.py | 28 ++++++++++++++++++++++++---- Lib/test/test_subprocess.py | 14 ++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 4638485975f1a3..11d50fe6b36071 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -43,6 +43,7 @@ import builtins import errno import io +import locale import os import time import signal @@ -344,6 +345,26 @@ def _args_from_interpreter_flags(): return args +def _text_encoding(): + # Return default text encoding and emit EncodingWarning if + # sys.flags.warn_default_encoding is true. + if sys.flags.warn_default_encoding: + f = sys._getframe() + filename = f.f_code.co_filename + stacklevel = 2 + while f := f.f_back: + if f.f_code.co_filename != filename: + break + stacklevel += 1 + warnings.warn("'encoding' argument not specified.", + EncodingWarning, stacklevel) + + if sys.flags.utf8_mode: + return "utf-8" + else: + return locale.getencoding() + + def call(*popenargs, timeout=None, **kwargs): """Run command with arguments. Wait for command to complete or timeout, then return the returncode attribute. @@ -610,7 +631,7 @@ def list2cmdline(seq): # Various tools for executing commands and looking at their output and status. # -def getstatusoutput(cmd): +def getstatusoutput(cmd, *, encoding=None): """Return (exitcode, output) of executing cmd in a shell. Execute the string 'cmd' in a shell with 'check_output' and @@ -858,9 +879,8 @@ def __init__(self, args, bufsize=-1, executable=None, errread = msvcrt.open_osfhandle(errread.Detach(), 0) self.text_mode = encoding or errors or text or universal_newlines - if self.text_mode: - # Emit EncodingWarning when text mode but *encoding* is not specified. - self.encoding = encoding = io.text_encoding(encoding) + if self.text_mode and encoding is None: + self.encoding = encoding = _text_encoding() # How long to resume waiting on a child after the first ^C. # There is no right value for this. The purpose is to be polite diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 99b5947e54be68..5814a6d924e128 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1733,6 +1733,20 @@ def test_run_with_shell_timeout_and_capture_output(self): msg="TimeoutExpired was delayed! Bad traceback:\n```\n" f"{stacks}```") + def test_encoding_warning(self): + code = textwrap.dedent("""\ + from subprocess import * + args = ["echo", "hello"] + run(args, text=True) + check_output(args, text=True) + """) + cp = subprocess.run([sys.executable, "-Xwarn_default_encoding", "-c", code], + capture_output=True) + lines = cp.stderr.splitlines() + self.assertEqual(len(lines), 2) + self.assertTrue(lines[0].startswith(b":3: EncodingWarning: ")) + self.assertTrue(lines[1].startswith(b":4: EncodingWarning: ")) + def _get_test_grp_name(): for name_group in ('staff', 'nogroup', 'grp', 'nobody', 'nfsnobody'): From 25e979c841ace800598da944c430a808ecb2684b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 13:13:32 +0900 Subject: [PATCH 5/8] Add encoding option to getoutput() --- Lib/subprocess.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 11d50fe6b36071..8dbb5ae481b65d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -662,7 +662,7 @@ def getstatusoutput(cmd, *, encoding=None): data = data[:-1] return exitcode, data -def getoutput(cmd): +def getoutput(cmd, *, encoding=None): """Return output (stdout or stderr) of executing cmd in a shell. Like getstatusoutput(), except the exit status is ignored and the return @@ -672,7 +672,8 @@ def getoutput(cmd): >>> subprocess.getoutput('ls /bin/ls') '/bin/ls' """ - return getstatusoutput(cmd)[1] + return getstatusoutput(cmd, encoding=encoding)[1] + def _use_posix_spawn(): From 66d176d02f40177346ff7ada0909a826c53bfdcb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 13:24:35 +0900 Subject: [PATCH 6/8] Add `errors` too --- Doc/library/subprocess.rst | 12 +++++++++--- Lib/subprocess.py | 9 +++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index fca55496492685..5ce32457e506fd 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1445,12 +1445,13 @@ This module also provides the following legacy functions from the 2.x none of the guarantees described above regarding security and exception handling consistency are valid for these functions. -.. function:: getstatusoutput(cmd) +.. function:: getstatusoutput(cmd, *, encoding=None, errors=None) Return ``(exitcode, output)`` of executing *cmd* in a shell. Execute the string *cmd* in a shell with :meth:`Popen.check_output` and - return a 2-tuple ``(exitcode, output)``. The locale encoding is used; + return a 2-tuple ``(exitcode, output)``. + *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. A trailing newline is stripped from the output. @@ -1475,8 +1476,10 @@ handling consistency are valid for these functions. as it did in Python 3.3.3 and earlier. exitcode has the same value as :attr:`~Popen.returncode`. + .. versionadded:: 3.11 + Added *encoding* and *errors* arguments. -.. function:: getoutput(cmd) +.. function:: getoutput(cmd, *, encoding=None, errors=None) Return output (stdout and stderr) of executing *cmd* in a shell. @@ -1491,6 +1494,9 @@ handling consistency are valid for these functions. .. versionchanged:: 3.3.4 Windows support added + .. versionadded:: 3.11 + Added *encoding* and *errors* arguments. + Notes ----- diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 8dbb5ae481b65d..968cfc14ddf910 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -631,7 +631,7 @@ def list2cmdline(seq): # Various tools for executing commands and looking at their output and status. # -def getstatusoutput(cmd, *, encoding=None): +def getstatusoutput(cmd, *, encoding=None, errors=None): """Return (exitcode, output) of executing cmd in a shell. Execute the string 'cmd' in a shell with 'check_output' and @@ -653,7 +653,8 @@ def getstatusoutput(cmd, *, encoding=None): (-15, '') """ try: - data = check_output(cmd, shell=True, text=True, stderr=STDOUT) + data = check_output(cmd, shell=True, text=True, stderr=STDOUT, + encoding=encoding, errors=errors) exitcode = 0 except CalledProcessError as ex: data = ex.output @@ -662,7 +663,7 @@ def getstatusoutput(cmd, *, encoding=None): data = data[:-1] return exitcode, data -def getoutput(cmd, *, encoding=None): +def getoutput(cmd, *, encoding=None, errors=None): """Return output (stdout or stderr) of executing cmd in a shell. Like getstatusoutput(), except the exit status is ignored and the return @@ -672,7 +673,7 @@ def getoutput(cmd, *, encoding=None): >>> subprocess.getoutput('ls /bin/ls') '/bin/ls' """ - return getstatusoutput(cmd, encoding=encoding)[1] + return getstatusoutput(cmd, encoding=encoding, errors=errors)[1] From d7cced5cde5382c054f8ebd3ae32fc102a88335c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 13:28:17 +0900 Subject: [PATCH 7/8] fix formatting in subprocess doc --- Doc/library/subprocess.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 5ce32457e506fd..6a334acc5a17cc 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -619,7 +619,7 @@ functions. If *encoding* or *errors* are specified, or *text* is true, the file objects *stdin*, *stdout* and *stderr* are opened in text mode with the specified - encoding and *errors*, as described above in :ref:`frequently-used-arguments`. + *encoding* and *errors*, as described above in :ref:`frequently-used-arguments`. The *universal_newlines* argument is equivalent to *text* and is provided for backwards compatibility. By default, file objects are opened in binary mode. From b4d98134e62f9e4dc107e9f506c6fc37c2500a68 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Apr 2022 13:30:29 +0900 Subject: [PATCH 8/8] Add NEWS --- .../next/Library/2022-04-27-13-30-26.gh-issue-91954.cC7ga_.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-04-27-13-30-26.gh-issue-91954.cC7ga_.rst diff --git a/Misc/NEWS.d/next/Library/2022-04-27-13-30-26.gh-issue-91954.cC7ga_.rst b/Misc/NEWS.d/next/Library/2022-04-27-13-30-26.gh-issue-91954.cC7ga_.rst new file mode 100644 index 00000000000000..b63db25f32a4f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-27-13-30-26.gh-issue-91954.cC7ga_.rst @@ -0,0 +1,2 @@ +Add *encoding* and *errors* arguments to :func:`subprocess.getoutput` and +:func:`subprocess.getstatusoutput`.