From 174953136674908ae6b5b6a4d43ce7f547d35fcd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 9 Jun 2025 11:30:24 +0100 Subject: [PATCH 1/3] Backport 135115 --- Lib/test/test_generators.py | 21 +++++++++------ Lib/test/test_genexps.py | 8 ++++++ ...-06-09-11-29-57.gh-issue-135171.1y8ObZ.rst | 12 +++++++++ Python/bytecodes.c | 27 ++++++++++++++++--- Python/compile.c | 11 +++++--- Python/executor_cases.c.h | 9 ++++++- Python/generated_cases.c.h | 18 +++++++++++-- 7 files changed, 89 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-11-29-57.gh-issue-135171.1y8ObZ.rst diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index a6c1dd62a23cc3..95c368a58ab17b 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -296,21 +296,26 @@ def gen(it): yield x return gen(range(10)) - def process_tests(self, get_generator): + def process_tests(self, get_generator, is_expr): + err_iterator = "'.*' object is not an iterator" + err_iterable = "'.*' object is not iterable" for obj in self.iterables: g_obj = get_generator(obj) with self.subTest(g_obj=g_obj, obj=obj): - self.assertListEqual(list(g_obj), list(obj)) + if is_expr: + self.assertRaisesRegex(TypeError, err_iterator, list, g_obj) + else: + self.assertListEqual(list(g_obj), list(obj)) g_iter = get_generator(iter(obj)) with self.subTest(g_iter=g_iter, obj=obj): self.assertListEqual(list(g_iter), list(obj)) - err_regex = "'.*' object is not iterable" for obj in self.non_iterables: g_obj = get_generator(obj) with self.subTest(g_obj=g_obj): - self.assertRaisesRegex(TypeError, err_regex, list, g_obj) + err = err_iterator if is_expr else err_iterable + self.assertRaisesRegex(TypeError, err, list, g_obj) def test_modify_f_locals(self): def modify_f_locals(g, local, obj): @@ -323,8 +328,8 @@ def get_generator_genexpr(obj): def get_generator_genfunc(obj): return modify_f_locals(self.genfunc(), 'it', obj) - self.process_tests(get_generator_genexpr) - self.process_tests(get_generator_genfunc) + self.process_tests(get_generator_genexpr, True) + self.process_tests(get_generator_genfunc, False) def test_new_gen_from_gi_code(self): def new_gen_from_gi_code(g, obj): @@ -337,8 +342,8 @@ def get_generator_genexpr(obj): def get_generator_genfunc(obj): return new_gen_from_gi_code(self.genfunc(), obj) - self.process_tests(get_generator_genexpr) - self.process_tests(get_generator_genfunc) + self.process_tests(get_generator_genexpr, True) + self.process_tests(get_generator_genfunc, False) class ExceptionTest(unittest.TestCase): diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index bb241837bd6679..b3e52707308f0f 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -131,6 +131,14 @@ >>> list(g) [1, 9, 25, 49, 81] +Verify that the outermost for-expression makes an immediate check +for iterability + >>> (i for i in 6) + Traceback (most recent call last): + File "", line 1, in -toplevel- + (i for i in 6) + TypeError: 'int' object is not iterable + Verify late binding for the innermost for-expression >>> g = ((i,j) for i in range(3) for j in range(x)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-11-29-57.gh-issue-135171.1y8ObZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-11-29-57.gh-issue-135171.1y8ObZ.rst new file mode 100644 index 00000000000000..004c031664d280 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-11-29-57.gh-issue-135171.1y8ObZ.rst @@ -0,0 +1,12 @@ +Reverted change made in 3.13.4 where creating a generator from an object +that was not an iterable did not immediately raise an exception. Code like +this:: + + def is_iterable(x): + try: + (_ for _ in x) + return True + except TypeError + return False + +now works again. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1da434bbbc892a..3775b5b00ceefc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2599,7 +2599,14 @@ dummy_func( replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); + iternextfunc func = Py_TYPE(iter)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + ERROR_NO_POP(); + } + next = func(iter); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { @@ -2622,7 +2629,14 @@ dummy_func( op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); + iternextfunc func = Py_TYPE(iter)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + ERROR_NO_POP(); + } + next = func(iter); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { @@ -2643,7 +2657,14 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + iternextfunc func = Py_TYPE(iter)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + ERROR_NO_POP(); + } + next = func(iter); if (next != NULL) { PUSH(next); target = next_instr; diff --git a/Python/compile.c b/Python/compile.c index e9506d6d978d89..e4da4cb75f16b8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5368,7 +5368,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); - + int is_outer_genexpr = gen_index == 0 && type == COMP_GENEXP; if (!iter_on_stack) { if (gen_index == 0) { /* Receive outermost iter as an implicit argument */ @@ -5407,7 +5407,9 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, if (IS_LABEL(start)) { depth++; - ADDOP(c, LOC(gen->iter), GET_ITER); + if (!is_outer_genexpr) { + ADDOP(c, LOC(gen->iter), GET_ITER); + } USE_LABEL(c, start); ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor); } @@ -5810,6 +5812,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, location loc = LOC(e); outermost = (comprehension_ty) asdl_seq_GET(generators, 0); + int is_sync_genexpr = type == COMP_GENEXP && !outermost->is_async; if (is_inlined) { if (compiler_visit_expr(c, outermost->iter) < 0) { goto error; @@ -5898,7 +5901,9 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, Py_CLEAR(co); VISIT(c, expr, outermost->iter); - + if (is_sync_genexpr) { + ADDOP(c, loc, GET_ITER); + } ADDOP_I(c, loc, CALL, 0); if (is_async_generator && type != COMP_GENEXP) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1bfbeb675b1b6f..66a2e474748172 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2629,7 +2629,14 @@ PyObject *next; iter = stack_pointer[-1]; /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); + iternextfunc func = Py_TYPE(iter)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + JUMP_TO_ERROR(); + } + next = func(iter); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index be00bf6eb6a39e..12d170e735b419 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2787,7 +2787,14 @@ // _FOR_ITER { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); + iternextfunc func = Py_TYPE(iter)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + goto error; + } + next = func(iter); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { @@ -3303,7 +3310,14 @@ /* Skip 1 cache entry */ _Py_CODEUNIT *target; PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + iternextfunc func = Py_TYPE(iter)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + goto error; + } + next = func(iter); if (next != NULL) { PUSH(next); target = next_instr; From 9e24a5cf08460ec0b163feb7e095c4f5a4e29cc0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 9 Jun 2025 11:43:48 +0100 Subject: [PATCH 2/3] Fix copy-and-paste errors --- Python/bytecodes.c | 8 ++++---- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3775b5b00ceefc..bcfc78cf1d45c3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2603,7 +2603,7 @@ dummy_func( if (func == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'%.100s' object is not an iterator", - Py_TYPE(iter_o)->tp_name); + Py_TYPE(iter)->tp_name); ERROR_NO_POP(); } next = func(iter); @@ -2633,7 +2633,7 @@ dummy_func( if (func == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'%.100s' object is not an iterator", - Py_TYPE(iter_o)->tp_name); + Py_TYPE(iter)->tp_name); ERROR_NO_POP(); } next = func(iter); @@ -2661,10 +2661,10 @@ dummy_func( if (func == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'%.100s' object is not an iterator", - Py_TYPE(iter_o)->tp_name); + Py_TYPE(iter)->tp_name); ERROR_NO_POP(); } - next = func(iter); + PyObject *next = func(iter); if (next != NULL) { PUSH(next); target = next_instr; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 66a2e474748172..a5cbadadd23f22 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2633,7 +2633,7 @@ if (func == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'%.100s' object is not an iterator", - Py_TYPE(iter_o)->tp_name); + Py_TYPE(iter)->tp_name); JUMP_TO_ERROR(); } next = func(iter); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 12d170e735b419..0f3f9f8fcfc489 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2791,7 +2791,7 @@ if (func == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'%.100s' object is not an iterator", - Py_TYPE(iter_o)->tp_name); + Py_TYPE(iter)->tp_name); goto error; } next = func(iter); @@ -3314,10 +3314,10 @@ if (func == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'%.100s' object is not an iterator", - Py_TYPE(iter_o)->tp_name); + Py_TYPE(iter)->tp_name); goto error; } - next = func(iter); + PyObject *next = func(iter); if (next != NULL) { PUSH(next); target = next_instr; From 025d45368a1b9c37823a4eb0e88824a2e457041f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 9 Jun 2025 11:48:43 +0100 Subject: [PATCH 3/3] Update test_dis --- Lib/test/test_dis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 51e00418165467..665e307166d275 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -184,6 +184,7 @@ def bug1333982(x=[]): LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) + GET_ITER CALL 0 %3d LOAD_CONST 2 (1) @@ -764,6 +765,7 @@ def foo(x): MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) + GET_ITER CALL 0 CALL 1 RETURN_VALUE @@ -784,7 +786,6 @@ def foo(x): POP_TOP L1: RESUME 0 LOAD_FAST 0 (.0) - GET_ITER L2: FOR_ITER 10 (to L3) STORE_FAST 1 (z) LOAD_DEREF 2 (x)