From 714c3f9b798a413183b2bdf232499d1091f9a4b8 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 16:37:21 +0000 Subject: [PATCH 01/18] gh-130080: implement PEP 765 --- Include/internal/pycore_compile.h | 1 + Lib/test/test_except_star.py | 6 +- Lib/test/test_syntax.py | 159 +++++++++++++++++++++++------- Python/ast_opt.c | 135 +++++++++++++++++++++++-- Python/compile.c | 4 +- 5 files changed, 258 insertions(+), 47 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 9f0ca33892a43b..6dfd5598604cd3 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -36,6 +36,7 @@ extern int _PyCompile_AstOptimize( extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, + PyObject *filename, int optimize, int ff_features); diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index 284907f61213f8..47006c6d3a0c36 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -84,7 +84,8 @@ def test_break_in_except_star(self): if i == 2: break finally: - return 0 + pass + return 0 """) @@ -117,7 +118,8 @@ def test_continue_in_except_star_block_invalid(self): if i == 2: continue finally: - return 0 + pass + return 0 """) def test_return_in_except_star_block_invalid(self): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 6d06e6f6dcb566..24e30c7ce56e4a 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -846,7 +846,7 @@ SyntaxError: 'function call' is an illegal expression for augmented assignment -Test continue in finally in weird combinations. +Test control flow in finally continue in for loop under finally should be ok. @@ -860,51 +860,63 @@ >>> test() 9 -continue in a finally should be ok. +break in for loop under finally should be ok. >>> def test(): - ... for abc in range(10): - ... try: - ... pass - ... finally: - ... continue - ... print(abc) + ... try: + ... pass + ... finally: + ... for abc in range(10): + ... break + ... print(abc) >>> test() - 9 + 0 + +return in function under finally should be ok. >>> def test(): - ... for abc in range(10): - ... try: - ... pass - ... finally: - ... try: - ... continue - ... except: - ... pass - ... print(abc) + ... try: + ... pass + ... finally: + ... def f(): + ... return 42 + ... print(f()) >>> test() - 9 + 42 + +combine for loop and function def + +return in function under finally should be ok. >>> def test(): - ... for abc in range(10): - ... try: - ... pass - ... finally: - ... try: - ... pass - ... except: - ... continue - ... print(abc) + ... try: + ... pass + ... finally: + ... for i in range(10): + ... def f(): + ... return 42 + ... print(f()) >>> test() - 9 + 42 + + >>> def test(): + ... try: + ... pass + ... finally: + ... def f(): + ... for i in range(10): + ... return 42 + ... print(f()) + >>> test() + 42 A continue outside loop should not be allowed. >>> def foo(): ... try: - ... pass - ... finally: ... continue + ... finally: + ... pass Traceback (most recent call last): ... SyntaxError: 'continue' not properly in loop @@ -2381,7 +2393,88 @@ def f(x: *b) from test import support -class SyntaxTestCase(unittest.TestCase): +class SyntaxWarningTest(unittest.TestCase): + def check_warning(self, code, errtext, filename="", mode="exec"): + """Check that compiling code raises SyntaxWarning with errtext. + + errtest is a regular expression that must be present in the + text of the warning raised. + """ + with self.assertWarnsRegex(SyntaxWarning, errtext): + compile(code, filename, mode) + + def test_return_in_finally(self): + source = textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """) + self.check_warning(source, "'return' in a 'finally' block") + + source = textwrap.dedent(""" + def f(): + try: + pass + finally: + try: + return 42 + except: + pass + """) + self.check_warning(source, "'return' in a 'finally' block") + + source = textwrap.dedent(""" + def f(): + try: + pass + finally: + try: + pass + except: + return 42 + """) + self.check_warning(source, "'return' in a 'finally' block") + + def test_break_and_continue_in_finally(self): + for kw in ('break', 'continue'): + + source = textwrap.dedent(f""" + for abc in range(10): + try: + pass + finally: + {kw} + """) + self.check_warning(source, f"'{kw}' in a 'finally' block") + + source = textwrap.dedent(f""" + for abc in range(10): + try: + pass + finally: + try: + {kw} + except: + pass + """) + self.check_warning(source, f"'{kw}' in a 'finally' block") + + source = textwrap.dedent(f""" + for abc in range(10): + try: + pass + finally: + try: + pass + except: + {kw} + """) + self.check_warning(source, f"'{kw}' in a 'finally' block") + + +class SyntaxErrorTestCase(unittest.TestCase): def _check_error(self, code, errtext, filename="", mode="exec", subclass=None, @@ -2389,7 +2482,7 @@ def _check_error(self, code, errtext, """Check that compiling code raises SyntaxError with errtext. errtest is a regular expression that must be present in the - test of the exception raised. If subclass is specified it + text of the exception raised. If subclass is specified it is the expected subclass of SyntaxError (e.g. IndentationError). """ try: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 0b58e8cd2a2ced..ecf6d58785b93b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -7,12 +7,22 @@ #include "pycore_setobject.h" // _PySet_NextEntry() +/* See PEP 765 */ typedef struct { + bool in_finally; + bool in_funcdef; + bool in_loop; +} ControlFlowInFinallyState; + +typedef struct { + PyObject *filename; int optimize; int ff_features; int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ + + ControlFlowInFinallyState cf_finally; } _PyASTOptimizeState; #define ENTER_RECURSIVE(ST) \ @@ -29,6 +39,82 @@ typedef struct { --(ST)->recursion_depth; \ } while(0) + +static ControlFlowInFinallyState +overwrite_state(_PyASTOptimizeState *state, bool finally, bool funcdef, bool loop) +{ + ControlFlowInFinallyState saved = state->cf_finally; + state->cf_finally.in_finally = finally; + state->cf_finally.in_funcdef = funcdef; + state->cf_finally.in_loop = loop; + return saved; +} + +static int +restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyState *saved) +{ + state->cf_finally = *saved; + return 1; +} + +static int +control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTOptimizeState *state) +{ + PyObject *msg = PyUnicode_FromFormat("'%s' in a 'finally' block", kw); + if (msg == NULL) { + return 0; + } + int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, + n->col_offset + 1, n->end_lineno, + n->end_col_offset + 1); + Py_DECREF(msg); + return ret < 0 ? 0 : 1; +} + +static int +before_return(_PyASTOptimizeState *state, stmt_ty node_) +{ + if (state->cf_finally.in_finally && ! state->cf_finally.in_funcdef) { + if (!control_flow_in_finally_warning("return", node_, state)) { + return 0; + } + } + return 1; +} + +static int +before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) +{ + if (state->cf_finally.in_finally && ! state->cf_finally.in_loop) { + if (!control_flow_in_finally_warning(kw, node_, state)) { + return 0; + } + } + return 1; +} + +#define RESTORE_STATE_CHECKED(S, CFS) \ + if (!restore_state((S), (CFS))) { \ + return 0; \ + } + +#define BEFORE_FINALLY(S) overwrite_state((S), true, false, false) +#define AFTER_FINALLY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define BEFORE_FUNC_BODY(S) overwrite_state((S), false, true, false) +#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define BEFORE_LOOP_BODY(S) overwrite_state((S), false, false, true) +#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) + +#define BEFORE_RETURN(S, N) \ + if (!before_return((S), (N))) { \ + return 0; \ + } + +#define BEFORE_LOOP_EXIT(S, N, KW) \ + if (!before_loop_exit((S), (N), (KW))) { \ + return 0; \ + } + static int make_const(expr_ty node, PyObject *val, PyArena *arena) { @@ -825,24 +911,30 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { ENTER_RECURSIVE(state); switch (node_->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); + ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); + AFTER_FUNC_BODY(state, &saved_state); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); } break; - case AsyncFunctionDef_kind: + } + case AsyncFunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); + ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); + AFTER_FUNC_BODY(state, &saved_state); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); } break; + } case ClassDef_kind: CALL_SEQ(astfold_type_param, type_param, node_->v.ClassDef.type_params); CALL_SEQ(astfold_expr, expr, node_->v.ClassDef.bases); @@ -851,6 +943,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_expr, expr, node_->v.ClassDef.decorator_list); break; case Return_kind: + BEFORE_RETURN(state, node_); CALL_OPT(astfold_expr, expr_ty, node_->v.Return.value); break; case Delete_kind: @@ -876,23 +969,32 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_type_param, type_param, node_->v.TypeAlias.type_params); CALL(astfold_expr, expr_ty, node_->v.TypeAlias.value); break; - case For_kind: + case For_kind: { CALL(astfold_expr, expr_ty, node_->v.For.target); CALL(astfold_expr, expr_ty, node_->v.For.iter); + ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); + AFTER_LOOP_BODY(state, &saved_state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); break; - case AsyncFor_kind: + } + case AsyncFor_kind: { CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); CALL(astfold_expr, expr_ty, node_->v.AsyncFor.iter); + ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.body); + AFTER_LOOP_BODY(state, &saved_state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.orelse); break; - case While_kind: + } + case While_kind: { CALL(astfold_expr, expr_ty, node_->v.While.test); + ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.body); + AFTER_LOOP_BODY(state, &saved_state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.orelse); break; + } case If_kind: CALL(astfold_expr, expr_ty, node_->v.If.test); CALL_SEQ(astfold_stmt, stmt, node_->v.If.body); @@ -910,18 +1012,24 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_OPT(astfold_expr, expr_ty, node_->v.Raise.exc); CALL_OPT(astfold_expr, expr_ty, node_->v.Raise.cause); break; - case Try_kind: + case Try_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.Try.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.Try.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.orelse); + ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.finalbody); + AFTER_FINALLY(state, &saved_state); break; - case TryStar_kind: + } + case TryStar_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.TryStar.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.orelse); + ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.finalbody); + AFTER_FINALLY(state, &saved_state); break; + } case Assert_kind: CALL(astfold_expr, expr_ty, node_->v.Assert.test); CALL_OPT(astfold_expr, expr_ty, node_->v.Assert.msg); @@ -933,14 +1041,18 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(astfold_expr, expr_ty, node_->v.Match.subject); CALL_SEQ(astfold_match_case, match_case, node_->v.Match.cases); break; + case Break_kind: + BEFORE_LOOP_EXIT(state, node_, "break"); + break; + case Continue_kind: + BEFORE_LOOP_EXIT(state, node_, "continue"); + break; // The following statements don't contain any subexpressions to be folded case Import_kind: case ImportFrom_kind: case Global_kind: case Nonlocal_kind: case Pass_kind: - case Break_kind: - case Continue_kind: break; // No default case, so the compiler will emit a warning if new statement // kinds are added without being handled here @@ -1045,12 +1157,15 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ int -_PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) +_PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, + int ff_features) { PyThreadState *tstate; int starting_recursion_depth; _PyASTOptimizeState state; + memset(&state, 0, sizeof(_PyASTOptimizeState)); + state.filename = filename; state.optimize = optimize; state.ff_features = ff_features; diff --git a/Python/compile.c b/Python/compile.c index b58c12d4b881ac..18332ada307958 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -126,7 +126,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Optimize(mod, arena, c->c_optimize, merged)) { + if (!_PyAST_Optimize(mod, arena, filename, c->c_optimize, merged)) { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -1397,7 +1397,7 @@ _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + if (!_PyAST_Optimize(mod, arena, filename, optimize, flags)) { return -1; } return 0; From 830fcfd63c70f0f0b9e390dbe15d15cab29f0355 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 19:08:08 +0000 Subject: [PATCH 02/18] add docs --- Doc/reference/compound_stmts.rst | 7 ++++++- Doc/tutorial/errors.rst | 8 ++++++-- Doc/whatsnew/3.14.rst | 10 ++++++++++ .../2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 71cc0c83de567e..30bf28e0d1af74 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -420,7 +420,7 @@ is executed. If there is a saved exception it is re-raised at the end of the :keyword:`!finally` clause. If the :keyword:`!finally` clause raises another exception, the saved exception is set as the context of the new exception. If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` -or :keyword:`continue` statement, the saved exception is discarded:: +or :keyword:`continue` statement, the saved exception is discarded. >>> def f(): ... try: @@ -461,6 +461,11 @@ always be the last one executed:: Prior to Python 3.8, a :keyword:`continue` statement was illegal in the :keyword:`!finally` clause due to a problem with the implementation. +.. versionchanged:: 3.14 + The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, + :keyword:`break` or :keyword:`continue` appears in a :keyword:`!finally` + block (see :pep:`765`). + .. _with: .. _as: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index c01cb8c14a0360..28990f3a65a6c1 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -417,7 +417,9 @@ points discuss more complex cases when an exception occurs: * If the :keyword:`!finally` clause executes a :keyword:`break`, :keyword:`continue` or :keyword:`return` statement, exceptions are not - re-raised. + re-raised. This can be confusing and is therefore discouraged. From + version 3.14 the compiler emits a :exc:`SyntaxWarning` for it + (see :pep:`765`). * If the :keyword:`!try` statement reaches a :keyword:`break`, :keyword:`continue` or :keyword:`return` statement, the @@ -429,7 +431,9 @@ points discuss more complex cases when an exception occurs: statement, the returned value will be the one from the :keyword:`!finally` clause's :keyword:`!return` statement, not the value from the :keyword:`!try` clause's :keyword:`!return` - statement. + statement. This can be confusing and is therefore discouraged. From + version 3.14 the compiler emits a :exc:`SyntaxWarning` for it + (see :pep:`765`). For example:: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ac0ae8cf0133e6..022ee5091f3325 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -68,6 +68,7 @@ Summary -- release highlights * :ref:`PEP 741: Python Configuration C API ` * :ref:`PEP 761: Discontinuation of PGP signatures ` * :ref:`A new type of interpreter ` +* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` New features @@ -307,6 +308,15 @@ Other language changes The testbed can also be used to run the test suite of projects other than CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) +.. _whatsnew314-pep765: + +PEP 765: Disallow return/break/continue that exit a finally block +----------------------------------------------------------------- + +The compiler emits a ``SyntaxWarning`` when a ``return``, ``break`` or +``continue`` statements appears where it exits a ``finally`` block. +This change in is specified in :pep:`765`. + New modules =========== diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst new file mode 100644 index 00000000000000..7c9f30a9f973f6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst @@ -0,0 +1 @@ +Implement PEP 765: Disallow return/break/continue that exit a finally block. From 3a8d59ea2306315a68faf2b09944d974a6ec1d82 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 19:27:22 +0000 Subject: [PATCH 03/18] doctest doesn't like it --- Doc/reference/compound_stmts.rst | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 30bf28e0d1af74..d4efd36c2db3b3 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -420,16 +420,15 @@ is executed. If there is a saved exception it is re-raised at the end of the :keyword:`!finally` clause. If the :keyword:`!finally` clause raises another exception, the saved exception is set as the context of the new exception. If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` -or :keyword:`continue` statement, the saved exception is discarded. +or :keyword:`continue` statement, the saved exception is discarded. For example, +this function returns 42. - >>> def f(): - ... try: - ... 1/0 - ... finally: - ... return 42 - ... - >>> f() - 42 +.. code:: + def f(): + try: + 1/0 + finally: + return 42 The exception information is not available to the program during execution of the :keyword:`!finally` clause. @@ -446,16 +445,14 @@ statement, the :keyword:`!finally` clause is also executed 'on the way out.' The return value of a function is determined by the last :keyword:`return` statement executed. Since the :keyword:`!finally` clause always executes, a :keyword:`!return` statement executed in the :keyword:`!finally` clause will -always be the last one executed:: +always be the last one executed. The following function returns 'finally'. - >>> def foo(): - ... try: - ... return 'try' - ... finally: - ... return 'finally' - ... - >>> foo() - 'finally' +.. code:: + def foo(): + try: + return 'try' + finally: + return 'finally' .. versionchanged:: 3.8 Prior to Python 3.8, a :keyword:`continue` statement was illegal in the From 51cdf5a20b926c356a4b80259086e8aa604bcbe5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 22:19:37 +0000 Subject: [PATCH 04/18] formatting --- Doc/reference/compound_stmts.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index d4efd36c2db3b3..5392c2358a6a43 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -424,6 +424,7 @@ or :keyword:`continue` statement, the saved exception is discarded. For example, this function returns 42. .. code:: + def f(): try: 1/0 @@ -448,6 +449,7 @@ statement executed. Since the :keyword:`!finally` clause always executes, a always be the last one executed. The following function returns 'finally'. .. code:: + def foo(): try: return 'try' From 480ac8efc0a3fafe57c6e3350ed73bdd76300cb7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 16 Feb 2025 13:47:58 +0000 Subject: [PATCH 05/18] always run ast optimizer --- Include/internal/pycore_compile.h | 6 ++-- Lib/test/test___all__.py | 1 + Python/ast_opt.c | 16 ++++++++- Python/bltinmodule.c | 54 +++++++++++++------------------ Python/compile.c | 6 ++-- Python/pythonrun.c | 9 +++--- 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 6dfd5598604cd3..63628af1a8f765 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -31,14 +31,16 @@ extern int _PyCompile_AstOptimize( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + int syntax_check_only); extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, PyObject *filename, int optimize, - int ff_features); + int ff_features, + int syntax_check_only); typedef struct { diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index e405056c8ffcb5..fd156a5dfa8572 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -37,6 +37,7 @@ def check_all(self, modname): (".* (module|package)", DeprecationWarning), (".* (module|package)", PendingDeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("import %s" % modname, names) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ecf6d58785b93b..57ada204bfb097 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -18,6 +18,7 @@ typedef struct { PyObject *filename; int optimize; int ff_features; + int syntax_check_only; int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ @@ -165,6 +166,9 @@ unary_not(PyObject *v) static int fold_unaryop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { + if (state->syntax_check_only) { + return 1; + } expr_ty arg = node->v.UnaryOp.operand; if (arg->kind != Constant_kind) { @@ -548,6 +552,9 @@ optimize_format(expr_ty node, PyObject *fmt, asdl_expr_seq *elts, PyArena *arena static int fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { + if (state->syntax_check_only) { + return 1; + } expr_ty lhs, rhs; lhs = node->v.BinOp.left; rhs = node->v.BinOp.right; @@ -644,6 +651,9 @@ make_const_tuple(asdl_expr_seq *elts) static int fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { + if (state->syntax_check_only) { + return 1; + } PyObject *newval; if (node->v.Tuple.ctx != Load) @@ -849,6 +859,9 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(fold_tuple, expr_ty, node_); break; case Name_kind: + if (state->syntax_check_only) { + break; + } if (node_->v.Name.ctx == Load && _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { LEAVE_RECURSIVE(state); @@ -1158,7 +1171,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat int _PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features) + int ff_features, int syntax_check_only) { PyThreadState *tstate; int starting_recursion_depth; @@ -1168,6 +1181,7 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, state.filename = filename; state.optimize = optimize; state.ff_features = ff_features; + state.syntax_check_only = syntax_check_only; /* Setup recursion depth check counters */ tstate = _PyThreadState_GET(); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a7243baa64c2a8..162fe11b3cb987 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -834,45 +834,35 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (is_ast == -1) goto error; if (is_ast) { - if ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST) { - if (PyAst_CheckMode(source, compile_mode) < 0) { - goto error; - } - // return an un-optimized AST - result = Py_NewRef(source); + PyArena *arena = _PyArena_New(); + if (arena == NULL) { + goto error; } - else { - // Return an optimized AST or code object - PyArena *arena = _PyArena_New(); - if (arena == NULL) { + if (flags & PyCF_ONLY_AST) { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); goto error; } - - if (flags & PyCF_ONLY_AST) { - mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); - goto error; - } - if (_PyCompile_AstOptimize(mod, filename, &cf, optimize, - arena) < 0) { - _PyArena_Free(arena); - goto error; - } - result = PyAST_mod2obj(mod); + int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ + if (_PyCompile_AstOptimize(mod, filename, &cf, optimize, + arena, syntax_check_only) < 0) { + _PyArena_Free(arena); + goto error; } - else { - mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); - goto error; - } - result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + result = PyAST_mod2obj(mod); + } + else { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + goto error; } - _PyArena_Free(arena); + result = (PyObject*)_PyAST_Compile(mod, filename, + &cf, optimize, arena); } + _PyArena_Free(arena); goto finally; } diff --git a/Python/compile.c b/Python/compile.c index 18332ada307958..6d0a48e2a7fb9b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -126,7 +126,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Optimize(mod, arena, filename, c->c_optimize, merged)) { + if (!_PyAST_Optimize(mod, arena, filename, c->c_optimize, merged, 0)) { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -1387,7 +1387,7 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena) + int optimize, PyArena *arena, int no_const_folding) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1397,7 +1397,7 @@ _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Optimize(mod, arena, filename, optimize, flags)) { + if (!_PyAST_Optimize(mod, arena, filename, optimize, flags, no_const_folding)) { return -1; } return 0; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 945e267ef72c6f..84150577f7649b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1450,11 +1450,10 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { - if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { - if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena) < 0) { - _PyArena_Free(arena); - return NULL; - } + int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ + if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + _PyArena_Free(arena); + return NULL; } PyObject *result = PyAST_mod2obj(mod); _PyArena_Free(arena); From 17a554f3c892b133f4b55a56c4866c9d3af97f76 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Feb 2025 08:54:47 +0000 Subject: [PATCH 06/18] fix test_unparse --- Lib/test/test_unparse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index f45a651c7ccb5d..9efea1e037f447 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -422,9 +422,11 @@ def test_docstrings(self): self.check_ast_roundtrip(f"'''{docstring}'''") def test_constant_tuples(self): - self.check_src_roundtrip(ast.Module([ast.Constant(value=(1,))]), "(1,)") + locs = ast.fix_missing_locations self.check_src_roundtrip( - ast.Module([ast.Constant(value=(1, 2, 3))]), "(1, 2, 3)" + locs(ast.Module([ast.Expr(ast.Constant(value=(1,)))])), "(1,)") + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1, 2, 3)))])), "(1, 2, 3)" ) def test_function_type(self): From d344fba3c8b5e504f0cf1f2aca9f720b7e9ef048 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Feb 2025 10:31:48 +0000 Subject: [PATCH 07/18] add tests for ast.parse() --- Lib/test/test_ast/test_ast.py | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 42dbb6e79c33b4..de0287743ccc6b 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -820,6 +820,61 @@ def test_repr_large_input_crash(self): r"Exceeds the limit \(\d+ digits\)"): repr(ast.Constant(value=eval(source))) + def test_pep_765_warnings(self): + srcs = [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ] + for src in srcs: + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + ast.parse(src) + + def test_pep_765_no_warnings(self): + srcs = [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ] + for src in srcs: + ast.parse(src) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" From 0994bf95351968c9f45822930c8fe9a0f8298cc7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Feb 2025 15:57:19 +0000 Subject: [PATCH 08/18] update test___all__ --- Lib/test/test___all__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index fd156a5dfa8572..f35b1194308262 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -53,6 +53,7 @@ def check_all(self, modname): with warnings_helper.check_warnings( ("", DeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("from %s import *" % modname, names) From df8f210b4a32b9e594938adf0eadba0994a5d551 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 4 Mar 2025 16:52:56 +0000 Subject: [PATCH 09/18] rename State->Context. no return value --- Python/ast_opt.c | 52 ++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 57ada204bfb097..eee5cd7815ae75 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -12,7 +12,7 @@ typedef struct { bool in_finally; bool in_funcdef; bool in_loop; -} ControlFlowInFinallyState; +} ControlFlowInFinallyContext; typedef struct { PyObject *filename; @@ -23,7 +23,7 @@ typedef struct { int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - ControlFlowInFinallyState cf_finally; + ControlFlowInFinallyContext cf_finally; } _PyASTOptimizeState; #define ENTER_RECURSIVE(ST) \ @@ -41,21 +41,20 @@ typedef struct { } while(0) -static ControlFlowInFinallyState +static ControlFlowInFinallyContext overwrite_state(_PyASTOptimizeState *state, bool finally, bool funcdef, bool loop) { - ControlFlowInFinallyState saved = state->cf_finally; + ControlFlowInFinallyContext saved = state->cf_finally; state->cf_finally.in_finally = finally; state->cf_finally.in_funcdef = funcdef; state->cf_finally.in_loop = loop; return saved; } -static int -restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyState *saved) +static void +restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyContext *saved) { state->cf_finally = *saved; - return 1; } static int @@ -94,17 +93,14 @@ before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) return 1; } -#define RESTORE_STATE_CHECKED(S, CFS) \ - if (!restore_state((S), (CFS))) { \ - return 0; \ - } +#define RESTORE_STATE(S, CFS) restore_state((S), (CFS)) #define BEFORE_FINALLY(S) overwrite_state((S), true, false, false) -#define AFTER_FINALLY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define AFTER_FINALLY(S, CFS) RESTORE_STATE((S), (CFS)) #define BEFORE_FUNC_BODY(S) overwrite_state((S), false, true, false) -#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE((S), (CFS)) #define BEFORE_LOOP_BODY(S) overwrite_state((S), false, false, true) -#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE((S), (CFS)) #define BEFORE_RETURN(S, N) \ if (!before_return((S), (N))) { \ @@ -927,9 +923,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case FunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); - ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); - AFTER_FUNC_BODY(state, &saved_state); + AFTER_FUNC_BODY(state, &saved_context); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); @@ -939,9 +935,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case AsyncFunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); - ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); - AFTER_FUNC_BODY(state, &saved_state); + AFTER_FUNC_BODY(state, &saved_context); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); @@ -985,26 +981,26 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case For_kind: { CALL(astfold_expr, expr_ty, node_->v.For.target); CALL(astfold_expr, expr_ty, node_->v.For.iter); - ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); - AFTER_LOOP_BODY(state, &saved_state); + AFTER_LOOP_BODY(state, &saved_context); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); break; } case AsyncFor_kind: { CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); CALL(astfold_expr, expr_ty, node_->v.AsyncFor.iter); - ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.body); - AFTER_LOOP_BODY(state, &saved_state); + AFTER_LOOP_BODY(state, &saved_context); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.orelse); break; } case While_kind: { CALL(astfold_expr, expr_ty, node_->v.While.test); - ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.body); - AFTER_LOOP_BODY(state, &saved_state); + AFTER_LOOP_BODY(state, &saved_context); CALL_SEQ(astfold_stmt, stmt, node_->v.While.orelse); break; } @@ -1029,18 +1025,18 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_stmt, stmt, node_->v.Try.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.Try.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.orelse); - ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.finalbody); - AFTER_FINALLY(state, &saved_state); + AFTER_FINALLY(state, &saved_context); break; } case TryStar_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.TryStar.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.orelse); - ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.finalbody); - AFTER_FINALLY(state, &saved_state); + AFTER_FINALLY(state, &saved_context); break; } case Assert_kind: From ed2f4e7acf60df779420e6b8e4056cf2092df8b2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 5 Mar 2025 15:10:11 +0000 Subject: [PATCH 10/18] use stack for context --- Python/ast_opt.c | 94 +++++++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index eee5cd7815ae75..ca30c6fe223b6c 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -5,6 +5,7 @@ #include "pycore_long.h" // _PyLong #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() +#include "cpython/code.h" // CO_MAXBLOCKS /* See PEP 765 */ @@ -23,7 +24,8 @@ typedef struct { int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - ControlFlowInFinallyContext cf_finally; + int cf_finally_next; + ControlFlowInFinallyContext cf_finally[CO_MAXBLOCKS]; } _PyASTOptimizeState; #define ENTER_RECURSIVE(ST) \ @@ -41,20 +43,27 @@ typedef struct { } while(0) -static ControlFlowInFinallyContext -overwrite_state(_PyASTOptimizeState *state, bool finally, bool funcdef, bool loop) +static int +push_cf_context(_PyASTOptimizeState *state, stmt_ty node, bool finally, bool funcdef, bool loop) { - ControlFlowInFinallyContext saved = state->cf_finally; - state->cf_finally.in_finally = finally; - state->cf_finally.in_funcdef = funcdef; - state->cf_finally.in_loop = loop; - return saved; + if (state->cf_finally_next == CO_MAXBLOCKS) { + PyErr_SetString(PyExc_SyntaxError, "too many statically nested blocks"); + PyErr_RangedSyntaxLocationObject(state->filename, node->lineno, node->col_offset + 1, + node->end_lineno, node->end_col_offset + 1); + return 0; + } + ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next++]; + ctx->in_finally = finally; + ctx->in_funcdef = funcdef; + ctx->in_loop = loop; + return 1; } static void -restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyContext *saved) +pop_cf_context(_PyASTOptimizeState *state) { - state->cf_finally = *saved; + assert(state->cf_finally_next > 0); + state->cf_finally_next--; } static int @@ -74,9 +83,12 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTOptimizeState * static int before_return(_PyASTOptimizeState *state, stmt_ty node_) { - if (state->cf_finally.in_finally && ! state->cf_finally.in_funcdef) { - if (!control_flow_in_finally_warning("return", node_, state)) { - return 0; + if (state->cf_finally_next > 0) { + ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (ctx->in_finally && ! ctx->in_funcdef) { + if (!control_flow_in_finally_warning("return", node_, state)) { + return 0; + } } } return 1; @@ -85,22 +97,30 @@ before_return(_PyASTOptimizeState *state, stmt_ty node_) static int before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) { - if (state->cf_finally.in_finally && ! state->cf_finally.in_loop) { - if (!control_flow_in_finally_warning(kw, node_, state)) { - return 0; + if (state->cf_finally_next > 0) { + ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (ctx->in_finally && ! ctx->in_loop) { + if (!control_flow_in_finally_warning(kw, node_, state)) { + return 0; + } } } return 1; } -#define RESTORE_STATE(S, CFS) restore_state((S), (CFS)) +#define PUSH_CONTEXT(S, N, FINALLY, FUNCDEF, LOOP) \ + if (!push_cf_context((S), (N), (FINALLY), (FUNCDEF), (LOOP))) { \ + return 0; \ + } + +#define POP_CONTEXT(S) pop_cf_context(S) -#define BEFORE_FINALLY(S) overwrite_state((S), true, false, false) -#define AFTER_FINALLY(S, CFS) RESTORE_STATE((S), (CFS)) -#define BEFORE_FUNC_BODY(S) overwrite_state((S), false, true, false) -#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE((S), (CFS)) -#define BEFORE_LOOP_BODY(S) overwrite_state((S), false, false, true) -#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE((S), (CFS)) +#define BEFORE_FINALLY(S, N) PUSH_CONTEXT((S), (N), true, false, false) +#define AFTER_FINALLY(S) POP_CONTEXT(S) +#define BEFORE_FUNC_BODY(S, N) PUSH_CONTEXT((S), (N), false, true, false) +#define AFTER_FUNC_BODY(S) POP_CONTEXT(S) +#define BEFORE_LOOP_BODY(S, N) PUSH_CONTEXT((S), (N), false, false, true) +#define AFTER_LOOP_BODY(S) POP_CONTEXT(S) #define BEFORE_RETURN(S, N) \ if (!before_return((S), (N))) { \ @@ -923,9 +943,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case FunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); - ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); + BEFORE_FUNC_BODY(state, node_); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); - AFTER_FUNC_BODY(state, &saved_context); + AFTER_FUNC_BODY(state); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); @@ -935,9 +955,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case AsyncFunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); - ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); + BEFORE_FUNC_BODY(state, node_); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); - AFTER_FUNC_BODY(state, &saved_context); + AFTER_FUNC_BODY(state); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); @@ -981,26 +1001,26 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case For_kind: { CALL(astfold_expr, expr_ty, node_->v.For.target); CALL(astfold_expr, expr_ty, node_->v.For.iter); - ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); + BEFORE_LOOP_BODY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); - AFTER_LOOP_BODY(state, &saved_context); + AFTER_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); break; } case AsyncFor_kind: { CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); CALL(astfold_expr, expr_ty, node_->v.AsyncFor.iter); - ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); + BEFORE_LOOP_BODY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.body); - AFTER_LOOP_BODY(state, &saved_context); + AFTER_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.orelse); break; } case While_kind: { CALL(astfold_expr, expr_ty, node_->v.While.test); - ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); + BEFORE_LOOP_BODY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.While.body); - AFTER_LOOP_BODY(state, &saved_context); + AFTER_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.orelse); break; } @@ -1025,18 +1045,18 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_stmt, stmt, node_->v.Try.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.Try.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.orelse); - ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); + BEFORE_FINALLY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.finalbody); - AFTER_FINALLY(state, &saved_context); + AFTER_FINALLY(state); break; } case TryStar_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.TryStar.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.orelse); - ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); + BEFORE_FINALLY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.finalbody); - AFTER_FINALLY(state, &saved_context); + AFTER_FINALLY(state); break; } case Assert_kind: From 09e7fb403e08db50ff0def37a1309942ecce16bd Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 14 Mar 2025 10:58:07 +0000 Subject: [PATCH 11/18] use c_array instead of bounded array for context --- Python/ast_opt.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 51d08fecf0a14b..f44266e481824b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1,11 +1,11 @@ /* AST Optimizer */ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_c_array.h" // _Py_CArray_EnsureCapacity() #include "pycore_format.h" // F_LJUST #include "pycore_long.h" // _PyLong #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() -#include "cpython/code.h" // CO_MAXBLOCKS /* See PEP 765 */ @@ -24,8 +24,8 @@ typedef struct { int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - int cf_finally_next; - ControlFlowInFinallyContext cf_finally[CO_MAXBLOCKS]; + _Py_c_array_t cf_finally; /* context for PEP 678 check */ + int cf_finally_used; } _PyASTOptimizeState; #define ENTER_RECURSIVE() \ @@ -35,17 +35,23 @@ if (Py_EnterRecursiveCall(" during compilation")) { \ #define LEAVE_RECURSIVE() Py_LeaveRecursiveCall(); +static ControlFlowInFinallyContext* +get_cf_finally_top(_PyASTOptimizeState *state) +{ + int idx = state->cf_finally_used+1; + return state->cf_finally.array + idx * sizeof(ControlFlowInFinallyContext); +} static int push_cf_context(_PyASTOptimizeState *state, stmt_ty node, bool finally, bool funcdef, bool loop) { - if (state->cf_finally_next == CO_MAXBLOCKS) { - PyErr_SetString(PyExc_SyntaxError, "too many statically nested blocks"); - PyErr_RangedSyntaxLocationObject(state->filename, node->lineno, node->col_offset + 1, - node->end_lineno, node->end_col_offset + 1); + if (_Py_CArray_EnsureCapacity(&state->cf_finally, state->cf_finally_used+1) < 0) { return 0; } - ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next++]; + + state->cf_finally_used++; + ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); + ctx->in_finally = finally; ctx->in_funcdef = funcdef; ctx->in_loop = loop; @@ -55,8 +61,8 @@ push_cf_context(_PyASTOptimizeState *state, stmt_ty node, bool finally, bool fun static void pop_cf_context(_PyASTOptimizeState *state) { - assert(state->cf_finally_next > 0); - state->cf_finally_next--; + assert(state->cf_finally_used > 0); + state->cf_finally_used--; } static int @@ -76,8 +82,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTOptimizeState * static int before_return(_PyASTOptimizeState *state, stmt_ty node_) { - if (state->cf_finally_next > 0) { - ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (state->cf_finally_used > 0) { + ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_funcdef) { if (!control_flow_in_finally_warning("return", node_, state)) { return 0; @@ -90,8 +96,8 @@ before_return(_PyASTOptimizeState *state, stmt_ty node_) static int before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) { - if (state->cf_finally_next > 0) { - ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (state->cf_finally_used > 0) { + ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_loop) { if (!control_flow_in_finally_warning(kw, node_, state)) { return 0; @@ -978,9 +984,13 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; + if (_Py_CArray_Init(&state.cf_finally, sizeof(ControlFlowInFinallyContext), 20) < 0) { + return -1; + } int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); + _Py_CArray_Fini(&state.cf_finally); return ret; } From 1f5323c61cf8b350bd9404ed4b6d18a80cb153dd Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Mar 2025 18:27:25 +0000 Subject: [PATCH 12/18] fix crash --- Python/ast_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f44266e481824b..982e85a4ed33f7 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -38,7 +38,7 @@ if (Py_EnterRecursiveCall(" during compilation")) { \ static ControlFlowInFinallyContext* get_cf_finally_top(_PyASTOptimizeState *state) { - int idx = state->cf_finally_used+1; + int idx = state->cf_finally_used; return state->cf_finally.array + idx * sizeof(ControlFlowInFinallyContext); } From 7a9a3e1b70cb03ca9d6b7c6a52f6f2322433eb3c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sat, 15 Mar 2025 18:40:37 +0000 Subject: [PATCH 13/18] Apply suggestions from code review Co-authored-by: Tomas R. --- Doc/whatsnew/3.14.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9971bf19ff2eca..fe2023db2fc1fe 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -376,9 +376,9 @@ Other language changes PEP 765: Disallow return/break/continue that exit a finally block ----------------------------------------------------------------- -The compiler emits a ``SyntaxWarning`` when a ``return``, ``break`` or -``continue`` statements appears where it exits a ``finally`` block. -This change in is specified in :pep:`765`. +The compiler emits a `:exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or +:keyword:`continue` statements appears where it exits a :keyword:`!finally` block. +This change is specified in :pep:`765`. New modules =========== From 6a37d813f3ba67d551016642b0f92601c01ea231 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Mar 2025 19:09:42 +0000 Subject: [PATCH 14/18] typo --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index fe2023db2fc1fe..e962fdbef629cd 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -376,7 +376,7 @@ Other language changes PEP 765: Disallow return/break/continue that exit a finally block ----------------------------------------------------------------- -The compiler emits a `:exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or +The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or :keyword:`continue` statements appears where it exits a :keyword:`!finally` block. This change is specified in :pep:`765`. From 0b257c3970c735e91beccd073da916e4c0a436a4 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Mar 2025 20:11:29 +0000 Subject: [PATCH 15/18] fix windows issue --- Python/ast_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 982e85a4ed33f7..182424c50583a0 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -39,7 +39,7 @@ static ControlFlowInFinallyContext* get_cf_finally_top(_PyASTOptimizeState *state) { int idx = state->cf_finally_used; - return state->cf_finally.array + idx * sizeof(ControlFlowInFinallyContext); + return ((ControlFlowInFinallyContext*)state->cf_finally.array) + idx; } static int From a4b65f725d6ceba4909ee8ee14d3cebe34c1e368 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Mar 2025 17:14:36 +0000 Subject: [PATCH 16/18] removed unused --- Python/ast_opt.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 4819adb60ea164..4a191e919e412c 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -21,9 +21,6 @@ typedef struct { int ff_features; int syntax_check_only; - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ - _Py_c_array_t cf_finally; /* context for PEP 678 check */ int cf_finally_used; } _PyASTOptimizeState; From a00f297c51dcd4871f7f2ca70a9e266925cf5db0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:12:08 +0000 Subject: [PATCH 17/18] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/reference/compound_stmts.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 5392c2358a6a43..949cdf3be8b7e3 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -423,7 +423,7 @@ If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement, the saved exception is discarded. For example, this function returns 42. -.. code:: +.. code-block:: def f(): try: @@ -448,7 +448,7 @@ statement executed. Since the :keyword:`!finally` clause always executes, a :keyword:`!return` statement executed in the :keyword:`!finally` clause will always be the last one executed. The following function returns 'finally'. -.. code:: +.. code-block:: def foo(): try: @@ -460,7 +460,7 @@ always be the last one executed. The following function returns 'finally'. Prior to Python 3.8, a :keyword:`continue` statement was illegal in the :keyword:`!finally` clause due to a problem with the implementation. -.. versionchanged:: 3.14 +.. versionchanged:: next The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or :keyword:`continue` appears in a :keyword:`!finally` block (see :pep:`765`). From 459ccf0b9ad03c39436beff9f6f82633d5d74116 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:20:00 +0000 Subject: [PATCH 18/18] Update Doc/whatsnew/3.14.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f1916eee4a1003..789156974cb0d1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -377,7 +377,7 @@ PEP 765: Disallow return/break/continue that exit a finally block ----------------------------------------------------------------- The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or -:keyword:`continue` statements appears where it exits a :keyword:`!finally` block. +:keyword:`continue` statements appears where it exits a :keyword:`finally` block. This change is specified in :pep:`765`. New modules