From 984c815a1468bf81c69b09b107c43304994b14ff Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 30 Mar 2020 16:50:36 +0300 Subject: [PATCH 1/9] bpo-39562: Prevent collision of future and compiler flags --- Include/code.h | 20 ++++++++++---------- Lib/__future__.py | 16 ++++++++-------- Lib/test/test_future.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/Include/code.h b/Include/code.h index 107eba4b9c4314..c69b8fe4560502 100644 --- a/Include/code.h +++ b/Include/code.h @@ -90,17 +90,17 @@ typedef struct { /* These are no longer used. */ #if 0 -#define CO_GENERATOR_ALLOWED 0x1000 +#define CO_GENERATOR_ALLOWED 0x10000 #endif -#define CO_FUTURE_DIVISION 0x2000 -#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */ -#define CO_FUTURE_WITH_STATEMENT 0x8000 -#define CO_FUTURE_PRINT_FUNCTION 0x10000 -#define CO_FUTURE_UNICODE_LITERALS 0x20000 - -#define CO_FUTURE_BARRY_AS_BDFL 0x40000 -#define CO_FUTURE_GENERATOR_STOP 0x80000 -#define CO_FUTURE_ANNOTATIONS 0x100000 +#define CO_FUTURE_DIVISION 0x20000 +#define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */ +#define CO_FUTURE_WITH_STATEMENT 0x80000 +#define CO_FUTURE_PRINT_FUNCTION 0x100000 +#define CO_FUTURE_UNICODE_LITERALS 0x200000 + +#define CO_FUTURE_BARRY_AS_BDFL 0x400000 +#define CO_FUTURE_GENERATOR_STOP 0x800000 +#define CO_FUTURE_ANNOTATIONS 0x1000000 /* This value is found in the co_cell2arg array when the associated cell variable does not correspond to an argument. */ diff --git a/Lib/__future__.py b/Lib/__future__.py index e1135685d846ce..d7cb8ac5f49745 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -68,14 +68,14 @@ # this module. CO_NESTED = 0x0010 # nested_scopes CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) -CO_FUTURE_DIVISION = 0x2000 # division -CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default -CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement -CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function -CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals -CO_FUTURE_BARRY_AS_BDFL = 0x40000 -CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators -CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime +CO_FUTURE_DIVISION = 0x20000 # division +CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default +CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement +CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function +CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals +CO_FUTURE_BARRY_AS_BDFL = 0x400000 +CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators +CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index fdca2312fab7c2..e4f330c45b4293 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -1,5 +1,7 @@ # Test various flavors of legal and illegal future statements +import __future__ +import ast import unittest from test import support from textwrap import dedent @@ -75,6 +77,16 @@ def test_badfuture10(self): from test import badsyntax_future10 self.check_syntax_error(cm.exception, "badsyntax_future10", 3) + def test_ensure_flags_dont_clash(self): + flags = [ + getattr(__future__, future).compiler_flag + for future in __future__.all_feature_names + ] + flags.extend( + (ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, ast.PyCF_ONLY_AST, ast.PyCF_TYPE_COMMENTS) + ) + self.assertCountEqual(set(flags), flags) + def test_parserhack(self): # test that the parser.c::future_hack function works as expected # Note: although this test must pass, it's not testing the original From 8d6d207e6c123ba24445ed6effbb8e3c260c80b8 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 30 Mar 2020 17:20:14 +0300 Subject: [PATCH 2/9] Add support to compile for PyCF_ALLOW_TOP_LEVEL_AWAIT --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a9fc21f1087748..10e7c1770f5ed9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -739,7 +739,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, } if (flags & - ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_TYPE_COMMENTS)) + ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | PyCF_TYPE_COMMENTS)) { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags"); From ebf0003261a679065fb53fc02f0b5d89fc86d757 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 30 Mar 2020 17:28:21 +0300 Subject: [PATCH 3/9] Pack valid compile flags to a mask --- Include/compile.h | 3 ++- Python/bltinmodule.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/compile.h b/Include/compile.h index c0c73b29e4d9dd..b8b37a497986fb 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -24,7 +24,8 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_IGNORE_COOKIE 0x0800 #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 - +#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ + PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT) #ifndef Py_LIMITED_API typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 10e7c1770f5ed9..22ee5969473f52 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -739,7 +739,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, } if (flags & - ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | PyCF_TYPE_COMMENTS)) + ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_COMPILE_MASK)) { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags"); From c14888d76122f100e0158e58057a68c15a74db1d Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 30 Mar 2020 17:31:38 +0300 Subject: [PATCH 4/9] automatically discover compiler flags from ast --- Include/compile.h | 1 + Lib/test/test_future.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/compile.h b/Include/compile.h index b8b37a497986fb..bb60b89b8e695f 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -26,6 +26,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT) + #ifndef Py_LIMITED_API typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index e4f330c45b4293..12da81a1bce512 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -83,7 +83,7 @@ def test_ensure_flags_dont_clash(self): for future in __future__.all_feature_names ] flags.extend( - (ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, ast.PyCF_ONLY_AST, ast.PyCF_TYPE_COMMENTS) + getattr(ast, flag) for flag in dir(ast) if flag.startswith("PyCF_") ) self.assertCountEqual(set(flags), flags) From 7cd943a7d65e54628fd6898e1b78087d894eb93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Mon, 20 Apr 2020 20:41:28 +0300 Subject: [PATCH 5/9] Mention about change, describe tests, remove old macro --- Doc/whatsnew/3.9.rst | 3 +++ Include/code.h | 5 ++--- Lib/test/test_future.py | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index c4b49feed9fc1d..b63a7cc9447c40 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -827,6 +827,9 @@ Changes in the Python API in the object itself. (Contributed by Serhiy Storchaka in :issue:`40257`.) +* The constant values of future flags in the :mod:`future` module + is updated in order to prevent collision with compiler flags. + (Contributed by Batuhan Taskaya in :issue:`39562`) CPython bytecode changes ------------------------ diff --git a/Include/code.h b/Include/code.h index c69b8fe4560502..64fecc65e0c9cb 100644 --- a/Include/code.h +++ b/Include/code.h @@ -89,9 +89,8 @@ typedef struct { #define CO_ASYNC_GENERATOR 0x0200 /* These are no longer used. */ -#if 0 -#define CO_GENERATOR_ALLOWED 0x10000 -#endif +/* These values are changed in 3.9 to prevent collision with + compiler flags. See issue 39562 for more details. */ #define CO_FUTURE_DIVISION 0x20000 #define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */ #define CO_FUTURE_WITH_STATEMENT 0x80000 diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index 12da81a1bce512..7b65bf6a8896ce 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -78,10 +78,13 @@ def test_badfuture10(self): self.check_syntax_error(cm.exception, "badsyntax_future10", 3) def test_ensure_flags_dont_clash(self): + # test that future flags and compiler flags doesn't clash + # obtain future flags (CO_****) from future module flags = [ getattr(__future__, future).compiler_flag for future in __future__.all_feature_names ] + # obtain some of the exported compiler flags (PyCF_****) from ast flags.extend( getattr(ast, flag) for flag in dir(ast) if flag.startswith("PyCF_") ) From d1094d64848f54a33199699f7c5cdea1e91bee67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Tue, 21 Apr 2020 02:27:04 +0300 Subject: [PATCH 6/9] Apply suggestions Co-Authored-By: Victor Stinner --- Doc/whatsnew/3.9.rst | 2 +- Include/code.h | 4 ++-- Lib/test/test_future.py | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index b63a7cc9447c40..63fdb9e3a6444f 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -827,7 +827,7 @@ Changes in the Python API in the object itself. (Contributed by Serhiy Storchaka in :issue:`40257`.) -* The constant values of future flags in the :mod:`future` module +* The constant values of future flags in the :mod:`__future__` module is updated in order to prevent collision with compiler flags. (Contributed by Batuhan Taskaya in :issue:`39562`) diff --git a/Include/code.h b/Include/code.h index 64fecc65e0c9cb..8cf7554740b276 100644 --- a/Include/code.h +++ b/Include/code.h @@ -89,8 +89,8 @@ typedef struct { #define CO_ASYNC_GENERATOR 0x0200 /* These are no longer used. */ -/* These values are changed in 3.9 to prevent collision with - compiler flags. See issue 39562 for more details. */ +/* bpo-39562: These constant values are changed in Python 3.9 + to prevent collision with compiler flags. */ #define CO_FUTURE_DIVISION 0x20000 #define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */ #define CO_FUTURE_WITH_STATEMENT 0x80000 diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index 7b65bf6a8896ce..ec6adf64c91cfd 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -78,13 +78,14 @@ def test_badfuture10(self): self.check_syntax_error(cm.exception, "badsyntax_future10", 3) def test_ensure_flags_dont_clash(self): - # test that future flags and compiler flags doesn't clash - # obtain future flags (CO_****) from future module + # bpo-39562: test that future flags and compiler flags doesn't clash + + # obtain future flags (CO_***) from the __future__ module flags = [ getattr(__future__, future).compiler_flag for future in __future__.all_feature_names ] - # obtain some of the exported compiler flags (PyCF_****) from ast + # obtain some of the exported compiler flags (PyCF_***) from the ast module flags.extend( getattr(ast, flag) for flag in dir(ast) if flag.startswith("PyCF_") ) From 560a100ba25bb46ad8031aadf12fb969136b013c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Tue, 21 Apr 2020 02:28:41 +0300 Subject: [PATCH 7/9] Mention clashing variables, remove old comment --- Doc/whatsnew/3.9.rst | 3 ++- Include/code.h | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 63fdb9e3a6444f..6aba84b2ca01f7 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -828,7 +828,8 @@ Changes in the Python API (Contributed by Serhiy Storchaka in :issue:`40257`.) * The constant values of future flags in the :mod:`__future__` module - is updated in order to prevent collision with compiler flags. + is updated in order to prevent collision with compiler flags. Previously + ``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. (Contributed by Batuhan Taskaya in :issue:`39562`) CPython bytecode changes diff --git a/Include/code.h b/Include/code.h index 8cf7554740b276..b2e7f22f346cc2 100644 --- a/Include/code.h +++ b/Include/code.h @@ -88,7 +88,6 @@ typedef struct { #define CO_ITERABLE_COROUTINE 0x0100 #define CO_ASYNC_GENERATOR 0x0200 -/* These are no longer used. */ /* bpo-39562: These constant values are changed in Python 3.9 to prevent collision with compiler flags. */ #define CO_FUTURE_DIVISION 0x20000 From 5f8018bcc8b5a37cffaa5b9bd0d9d8e3bf4cead4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Tue, 21 Apr 2020 03:28:49 +0300 Subject: [PATCH 8/9] Add a noticement header, use dicts for tests --- Include/code.h | 4 +++- Include/compile.h | 5 +++++ Lib/test/test_future.py | 17 +++++++++-------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Include/code.h b/Include/code.h index b2e7f22f346cc2..b268a6aedff8d0 100644 --- a/Include/code.h +++ b/Include/code.h @@ -89,7 +89,9 @@ typedef struct { #define CO_ASYNC_GENERATOR 0x0200 /* bpo-39562: These constant values are changed in Python 3.9 - to prevent collision with compiler flags. */ + to prevent collision with compiler flags. CO_FUTURE_ and PyCF_ + constants must be kept unique. PyCF_ constants can use bits from + 0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */ #define CO_FUTURE_DIVISION 0x20000 #define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */ #define CO_FUTURE_WITH_STATEMENT 0x80000 diff --git a/Include/compile.h b/Include/compile.h index bb60b89b8e695f..fcf36e9f3e336e 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -18,6 +18,11 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) #define PyCF_MASK_OBSOLETE (CO_NESTED) + +/* bpo-39562: These constant values are changed in Python 3.9 + to prevent collision with compiler flags. CO_FUTURE_ and PyCF_ + constants must be kept unique. PyCF_ constants can use bits from + 0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */ #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 #define PyCF_ONLY_AST 0x0400 diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index ec6adf64c91cfd..56b7ac68655594 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -80,16 +80,17 @@ def test_badfuture10(self): def test_ensure_flags_dont_clash(self): # bpo-39562: test that future flags and compiler flags doesn't clash - # obtain future flags (CO_***) from the __future__ module - flags = [ - getattr(__future__, future).compiler_flag + # obtain future flags (CO_FUTURE_***) from the __future__ module + flags = { + f"CO_FUTURE_{future.upper()}": getattr(__future__, future).compiler_flag for future in __future__.all_feature_names - ] + } # obtain some of the exported compiler flags (PyCF_***) from the ast module - flags.extend( - getattr(ast, flag) for flag in dir(ast) if flag.startswith("PyCF_") - ) - self.assertCountEqual(set(flags), flags) + flags |= { + flag: getattr(ast, flag) + for flag in dir(ast) if flag.startswith("PyCF_") + } + self.assertCountEqual(set(flags.values()), flags.values()) def test_parserhack(self): # test that the parser.c::future_hack function works as expected From b225c616b64df49b9664f0213de4aabc0940e272 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 21 Apr 2020 02:59:53 +0200 Subject: [PATCH 9/9] Update Include/compile.h --- Include/compile.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/compile.h b/Include/compile.h index fcf36e9f3e336e..a2db65d47f001e 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -19,10 +19,9 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) #define PyCF_MASK_OBSOLETE (CO_NESTED) -/* bpo-39562: These constant values are changed in Python 3.9 - to prevent collision with compiler flags. CO_FUTURE_ and PyCF_ - constants must be kept unique. PyCF_ constants can use bits from - 0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */ +/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique. + PyCF_ constants can use bits from 0x0100 to 0x10000. + CO_FUTURE_ constants use bits starting at 0x20000. */ #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 #define PyCF_ONLY_AST 0x0400