Skip to content

Commit ad106c6

Browse files
isidenticalisidentical
authored
bpo-42725: Render annotations effectless on symbol table with PEP 563 (pythonGH-25583)
1 parent 37ebdf0 commit ad106c6

File tree

5 files changed

+149
-18
lines changed

5 files changed

+149
-18
lines changed

‎Include/internal/pycore_symtable.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ extern "C" {
1010

1111
struct _mod; // Type defined in pycore_ast.h
1212

13-
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
13+
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock }
1414
_Py_block_ty;
1515

1616
struct _symtable_entry;

‎Lib/test/test_future.py

+55-6
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,14 @@ def assertAnnotationEqual(
171171

172172
self.assertEqual(actual, expected)
173173

174+
def _exec_future(self, code):
175+
scope = {}
176+
exec(
177+
"from __future__ import annotations\n"
178+
+ code, {}, scope
179+
)
180+
return scope
181+
174182
def test_annotations(self):
175183
eq = self.assertAnnotationEqual
176184
eq('...')
@@ -310,19 +318,13 @@ def test_annotations(self):
310318
eq("f'{x}'")
311319
eq("f'{x!r}'")
312320
eq("f'{x!a}'")
313-
eq('(yield from outside_of_generator)')
314-
eq('(yield)')
315-
eq('(yield a + b)')
316-
eq('await some.complicated[0].call(with_args=True or 1 is not 1)')
317321
eq('[x for x in (a if b else c)]')
318322
eq('[x for x in a if (b if c else d)]')
319323
eq('f(x for x in a)')
320324
eq('f(1, (x for x in a))')
321325
eq('f((x for x in a), 2)')
322326
eq('(((a)))', 'a')
323327
eq('(((a, b)))', '(a, b)')
324-
eq("(x := 10)")
325-
eq("f'{(x := 10):=10}'")
326328
eq("1 + 2 + 3")
327329

328330
def test_fstring_debug_annotations(self):
@@ -354,6 +356,53 @@ def test_annotation_with_complex_target(self):
354356
"object.__debug__: int"
355357
)
356358

359+
def test_annotations_symbol_table_pass(self):
360+
namespace = self._exec_future(dedent("""
361+
from __future__ import annotations
362+
363+
def foo():
364+
outer = 1
365+
def bar():
366+
inner: outer = 1
367+
return bar
368+
"""))
369+
370+
foo = namespace.pop("foo")
371+
self.assertIsNone(foo().__closure__)
372+
self.assertEqual(foo.__code__.co_cellvars, ())
373+
self.assertEqual(foo().__code__.co_freevars, ())
374+
375+
def test_annotations_forbidden(self):
376+
with self.assertRaises(SyntaxError):
377+
self._exec_future("test: (yield)")
378+
379+
with self.assertRaises(SyntaxError):
380+
self._exec_future("test.test: (yield a + b)")
381+
382+
with self.assertRaises(SyntaxError):
383+
self._exec_future("test[something]: (yield from x)")
384+
385+
with self.assertRaises(SyntaxError):
386+
self._exec_future("def func(test: (yield from outside_of_generator)): pass")
387+
388+
with self.assertRaises(SyntaxError):
389+
self._exec_future("def test() -> (await y): pass")
390+
391+
with self.assertRaises(SyntaxError):
392+
self._exec_future("async def test() -> something((a := b)): pass")
393+
394+
with self.assertRaises(SyntaxError):
395+
self._exec_future("test: await some.complicated[0].call(with_args=True or 1 is not 1)")
396+
397+
with self.assertRaises(SyntaxError):
398+
self._exec_future("test: f'{(x := 10):=10}'")
399+
400+
with self.assertRaises(SyntaxError):
401+
self._exec_future(dedent("""\
402+
def foo():
403+
def bar(arg: (yield)): pass
404+
"""))
405+
357406

358407
if __name__ == "__main__":
359408
unittest.main()

‎Misc/NEWS.d/next/Core and Builtins/2021-04-25-05-40-51.bpo-42725.WGloYm.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Usage of ``await``/``yield``/``yield from`` and named expressions within an
2+
annotation is now forbidden when PEP 563 is activated.

‎Modules/symtablemodule.c

-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ _symtable_symtable_impl(PyObject *module, PyObject *source,
5858
}
5959
t = (PyObject *)st->st_top;
6060
Py_INCREF(t);
61-
PyMem_Free((void *)st->st_future);
6261
_PySymtable_Free(st);
6362
return t;
6463
}

‎Python/symtable.c

+91-10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#include "Python.h"
22
#include "pycore_ast.h" // identifier, stmt_ty
3-
#include "pycore_compile.h" // _Py_Mangle()
3+
#include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST()
44
#include "pycore_parser.h" // _PyParser_ASTFromString()
55
#include "pycore_pystate.h" // _PyThreadState_GET()
66
#include "pycore_symtable.h" // PySTEntryObject
@@ -45,6 +45,10 @@
4545
#define NAMED_EXPR_COMP_ITER_EXPR \
4646
"assignment expression cannot be used in a comprehension iterable expression"
4747

48+
#define ANNOTATION_NOT_ALLOWED \
49+
"'%s' can not be used within an annotation"
50+
51+
4852
static PySTEntryObject *
4953
ste_new(struct symtable *st, identifier name, _Py_block_ty block,
5054
void *key, int lineno, int col_offset,
@@ -209,17 +213,19 @@ static int symtable_visit_alias(struct symtable *st, alias_ty);
209213
static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
210214
static int symtable_visit_keyword(struct symtable *st, keyword_ty);
211215
static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
216+
static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
212217
static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
213218
static int symtable_implicit_arg(struct symtable *st, int pos);
214-
static int symtable_visit_annotations(struct symtable *st, arguments_ty, expr_ty);
219+
static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
215220
static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
216221
static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
217222
static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
223+
static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
218224

219225

220226
static identifier top = NULL, lambda = NULL, genexpr = NULL,
221227
listcomp = NULL, setcomp = NULL, dictcomp = NULL,
222-
__class__ = NULL;
228+
__class__ = NULL, _annotation = NULL;
223229

224230
#define GET_IDENTIFIER(VAR) \
225231
((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR)))
@@ -987,8 +993,17 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
987993
/* The entry is owned by the stack. Borrow it for st_cur. */
988994
Py_DECREF(ste);
989995
st->st_cur = ste;
996+
997+
/* Annotation blocks shouldn't have any affect on the symbol table since in
998+
* the compilation stage, they will all be transformed to strings. They are
999+
* only created if future 'annotations' feature is activated. */
1000+
if (block == AnnotationBlock) {
1001+
return 1;
1002+
}
1003+
9901004
if (block == ModuleBlock)
9911005
st->st_global = st->st_cur->ste_symbols;
1006+
9921007
if (prev) {
9931008
if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
9941009
return 0;
@@ -1190,7 +1205,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
11901205
VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
11911206
if (s->v.FunctionDef.args->kw_defaults)
11921207
VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
1193-
if (!symtable_visit_annotations(st, s->v.FunctionDef.args,
1208+
if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
11941209
s->v.FunctionDef.returns))
11951210
VISIT_QUIT(st, 0);
11961211
if (s->v.FunctionDef.decorator_list)
@@ -1273,7 +1288,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
12731288
else {
12741289
VISIT(st, expr, s->v.AnnAssign.target);
12751290
}
1276-
VISIT(st, expr, s->v.AnnAssign.annotation);
1291+
if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
1292+
VISIT_QUIT(st, 0);
1293+
}
1294+
12771295
if (s->v.AnnAssign.value) {
12781296
VISIT(st, expr, s->v.AnnAssign.value);
12791297
}
@@ -1422,7 +1440,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
14221440
if (s->v.AsyncFunctionDef.args->kw_defaults)
14231441
VISIT_SEQ_WITH_NULL(st, expr,
14241442
s->v.AsyncFunctionDef.args->kw_defaults);
1425-
if (!symtable_visit_annotations(st, s->v.AsyncFunctionDef.args,
1443+
if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
14261444
s->v.AsyncFunctionDef.returns))
14271445
VISIT_QUIT(st, 0);
14281446
if (s->v.AsyncFunctionDef.decorator_list)
@@ -1564,6 +1582,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
15641582
}
15651583
switch (e->kind) {
15661584
case NamedExpr_kind:
1585+
if (!symtable_raise_if_annotation_block(st, "named expression", e)) {
1586+
VISIT_QUIT(st, 0);
1587+
}
15671588
if(!symtable_handle_namedexpr(st, e))
15681589
VISIT_QUIT(st, 0);
15691590
break;
@@ -1624,15 +1645,24 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
16241645
VISIT_QUIT(st, 0);
16251646
break;
16261647
case Yield_kind:
1648+
if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1649+
VISIT_QUIT(st, 0);
1650+
}
16271651
if (e->v.Yield.value)
16281652
VISIT(st, expr, e->v.Yield.value);
16291653
st->st_cur->ste_generator = 1;
16301654
break;
16311655
case YieldFrom_kind:
1656+
if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1657+
VISIT_QUIT(st, 0);
1658+
}
16321659
VISIT(st, expr, e->v.YieldFrom.value);
16331660
st->st_cur->ste_generator = 1;
16341661
break;
16351662
case Await_kind:
1663+
if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
1664+
VISIT_QUIT(st, 0);
1665+
}
16361666
VISIT(st, expr, e->v.Await.value);
16371667
st->st_cur->ste_coroutine = 1;
16381668
break;
@@ -1780,6 +1810,24 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
17801810
return 1;
17811811
}
17821812

1813+
static int
1814+
symtable_visit_annotation(struct symtable *st, expr_ty annotation)
1815+
{
1816+
int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1817+
if (future_annotations &&
1818+
!symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock,
1819+
(void *)annotation, annotation->lineno,
1820+
annotation->col_offset, annotation->end_lineno,
1821+
annotation->end_col_offset)) {
1822+
VISIT_QUIT(st, 0);
1823+
}
1824+
VISIT(st, expr, annotation);
1825+
if (future_annotations && !symtable_exit_block(st)) {
1826+
VISIT_QUIT(st, 0);
1827+
}
1828+
return 1;
1829+
}
1830+
17831831
static int
17841832
symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
17851833
{
@@ -1798,8 +1846,15 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
17981846
}
17991847

18001848
static int
1801-
symtable_visit_annotations(struct symtable *st, arguments_ty a, expr_ty returns)
1849+
symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
18021850
{
1851+
int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1852+
if (future_annotations &&
1853+
!symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock,
1854+
(void *)o, o->lineno, o->col_offset, o->end_lineno,
1855+
o->end_col_offset)) {
1856+
VISIT_QUIT(st, 0);
1857+
}
18031858
if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
18041859
return 0;
18051860
if (a->args && !symtable_visit_argannotations(st, a->args))
@@ -1810,8 +1865,12 @@ symtable_visit_annotations(struct symtable *st, arguments_ty a, expr_ty returns)
18101865
VISIT(st, expr, a->kwarg->annotation);
18111866
if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
18121867
return 0;
1813-
if (returns)
1814-
VISIT(st, expr, returns);
1868+
if (future_annotations && !symtable_exit_block(st)) {
1869+
VISIT_QUIT(st, 0);
1870+
}
1871+
if (returns && !symtable_visit_annotation(st, returns)) {
1872+
VISIT_QUIT(st, 0);
1873+
}
18151874
return 1;
18161875
}
18171876

@@ -2033,6 +2092,21 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e)
20332092
e->v.DictComp.value);
20342093
}
20352094

2095+
static int
2096+
symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
2097+
{
2098+
if (st->st_cur->ste_type != AnnotationBlock) {
2099+
return 1;
2100+
}
2101+
2102+
PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
2103+
PyErr_RangedSyntaxLocationObject(st->st_filename,
2104+
e->lineno,
2105+
e->col_offset + 1,
2106+
e->end_lineno,
2107+
e->end_col_offset + 1);
2108+
return 0;
2109+
}
20362110

20372111
struct symtable *
20382112
_Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
@@ -2051,7 +2125,14 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
20512125
_PyArena_Free(arena);
20522126
return NULL;
20532127
}
2054-
st = _PySymtable_Build(mod, filename, 0);
2128+
PyFutureFeatures *future = _PyFuture_FromAST(mod, filename);
2129+
if (future == NULL) {
2130+
_PyArena_Free(arena);
2131+
return NULL;
2132+
}
2133+
future->ff_features |= flags->cf_flags;
2134+
st = _PySymtable_Build(mod, filename, future);
2135+
PyObject_Free((void *)future);
20552136
_PyArena_Free(arena);
20562137
return st;
20572138
}

0 commit comments

Comments
 (0)