Skip to content

Commit be8df3d

Browse files
RayaneCTXRayaneCTX
committed
py: Implement decorator syntax relaxation from CPython 3.9.
py/grammar.h: The grammar file hosts the core change associated with this pull request. The decorator rule is changed to match a namedexpr_test rule rather than a dotted_name rule. However, since the compiler currently relies on the dotted_name rule to easily detect built-in decorators (e.g. @micropython.bytecode), a new way of detecting these built-ins needed to be found: 1. One possibility is to have the grammar try to match a dotted_name first and a namedexpr_test second, but since the parser does not bactrack through lexical tokens, it is not obvious to me how to accomplish this. 2. Another possiblity is to write code in the compiler to visit the AST rooted at the namedexpr_test rule of a given decorator and determine whether said decorator is a built-in that way. This could potentially be an expensive operation and/or lead to a significant increase in code size. 3. [SELECTED] A third option is to introduce a grammar rule built_in_decorator that tries to match a "micropython" string, and backtracks into a namedexpr_test rule if there is no match. This relies on the fact that all built-in decorators start with "micropython". One possible concern is that any decorator starting with the "micropython" NAME token must match with a built-in decorator, otherwise a syntax error is issued. This is, however, congruent with current behavior: this is exactly what the compile_built_in_decorator function would do (before this commit). py/parse.c: Add a check to attempt to match the "micropython" string to a NAME token when parsing a built_in_decrator rule. The parser backtracks out of the rule if there is no match. py/compile.c: Since the compile_built_in_decorator function is not used anywhere else, I took the liberty to simplify this function to match the new grammar. tests/basics: Amend the decorator.py test with additional tests for syntax, type and name errors. Adds an expected output file so that this test works out-of-the-box with older versions of CPython than 3.9. tests/cmdline/cmd_parsetree.py.exp: Update the expected rule IDs in the expected output since the grammar is changed. tools/ci.sh: Exclude the decorator_full.py test from the minimal build since it uses several unsupported features (e.g. decimal/complex numbers). ports/qemu_arm/Makefile.test: Exclude the decorator_full.py test here as well. Signed-off-by: Rayane Chatrieux <rayane.chatrieux@gmail.com>
1 parent 71344c1 commit be8df3d

File tree

6 files changed

+116
-41
lines changed

6 files changed

+116
-41
lines changed

‎py/compile.c

Lines changed: 19 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -857,17 +857,8 @@ STATIC qstr compile_classdef_helper(compiler_t *comp, mp_parse_node_struct_t *pn
857857
}
858858

859859
// returns true if it was a built-in decorator (even if the built-in had an error)
860-
STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_parse_node_t *name_nodes, uint *emit_options) {
861-
if (MP_PARSE_NODE_LEAF_ARG(name_nodes[0]) != MP_QSTR_micropython) {
862-
return false;
863-
}
864-
865-
if (name_len != 2) {
866-
compile_syntax_error(comp, name_nodes[0], MP_ERROR_TEXT("invalid micropython decorator"));
867-
return true;
868-
}
869-
870-
qstr attr = MP_PARSE_NODE_LEAF_ARG(name_nodes[1]);
860+
STATIC bool compile_built_in_decorator(compiler_t *comp, mp_parse_node_t name_node, uint *emit_options) {
861+
qstr attr = MP_PARSE_NODE_LEAF_ARG(name_node);
871862
if (attr == MP_QSTR_bytecode) {
872863
*emit_options = MP_EMIT_OPT_BYTECODE;
873864
#if MICROPY_EMIT_NATIVE
@@ -888,17 +879,17 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par
888879
#endif
889880
#endif
890881
} else {
891-
compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid micropython decorator"));
882+
compile_syntax_error(comp, name_node, MP_ERROR_TEXT("invalid micropython decorator"));
892883
}
893884

894885
#if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER
895886
if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) {
896887
if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) {
897-
compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid arch"));
888+
compile_syntax_error(comp, name_node, MP_ERROR_TEXT("invalid arch"));
898889
}
899890
} else if (*emit_options == MP_EMIT_OPT_ASM) {
900891
if (emit_asm_table[mp_dynamic_compiler.native_arch] == NULL) {
901-
compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid arch"));
892+
compile_syntax_error(comp, name_node, MP_ERROR_TEXT("invalid arch"));
902893
}
903894
}
904895
#endif
@@ -920,30 +911,22 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) {
920911
assert(MP_PARSE_NODE_IS_STRUCT_KIND(nodes[i], PN_decorator)); // should be
921912
mp_parse_node_struct_t *pns_decorator = (mp_parse_node_struct_t *)nodes[i];
922913

923-
// nodes[0] contains the decorator function, which is a dotted name
924-
mp_parse_node_t *name_nodes;
925-
size_t name_len = mp_parse_node_extract_list(&pns_decorator->nodes[0], PN_dotted_name, &name_nodes);
926-
927-
// check for built-in decorators
928-
if (compile_built_in_decorator(comp, name_len, name_nodes, &emit_options)) {
929-
// this was a built-in
930-
num_built_in_decorators += 1;
914+
assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns_decorator) == 1);
915+
mp_parse_node_t decorator_arg = pns_decorator->nodes[0];
931916

932-
} else {
933-
// not a built-in, compile normally
934-
935-
// compile the decorator function
936-
compile_node(comp, name_nodes[0]);
937-
for (size_t j = 1; j < name_len; j++) {
938-
assert(MP_PARSE_NODE_IS_ID(name_nodes[j])); // should be
939-
EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(name_nodes[j]), MP_EMIT_ATTR_LOAD);
940-
}
941-
942-
// nodes[1] contains arguments to the decorator function, if any
943-
if (!MP_PARSE_NODE_IS_NULL(pns_decorator->nodes[1])) {
944-
// call the decorator function with the arguments in nodes[1]
945-
compile_node(comp, pns_decorator->nodes[1]);
917+
if (MP_PARSE_NODE_IS_STRUCT_KIND(decorator_arg, PN_built_in_decorator)) {
918+
mp_parse_node_struct_t *decorator_arg_s = (mp_parse_node_struct_t *)decorator_arg;
919+
assert(MP_PARSE_NODE_STRUCT_NUM_NODES(decorator_arg_s) == 2);
920+
// The first child node must be the ID "micropython" (enforced by
921+
// the parser). We just need to have a look at the second child.
922+
mp_parse_node_t name_node = decorator_arg_s->nodes[1];
923+
if (compile_built_in_decorator(comp, name_node, &emit_options)) {
924+
num_built_in_decorators += 1;
946925
}
926+
} else {
927+
// decorator_arg is something derived from a namedexpr_test rule.
928+
// Compile it normally.
929+
compile_node(comp, decorator_arg);
947930
}
948931
}
949932

‎py/grammar.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@ DEF_RULE_NC(file_input_3, or(2), tok(NEWLINE), rule(stmt))
5151
DEF_RULE_NC(eval_input, and_ident(2), rule(testlist), opt_rule(eval_input_2))
5252
DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE))
5353

54-
// decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
54+
// decorator: '@' decorator_arg NEWLINE
55+
// decorator_arg: built_in_decorator | namedexpr_test
56+
// built_in_decorator: 'micropython' '.' NAME
5557
// decorators: decorator+
5658
// decorated: decorators (classdef | funcdef | async_funcdef)
5759
// funcdef: 'def' NAME parameters ['->' test] ':' suite
@@ -62,7 +64,9 @@ DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE))
6264
// varargslist: vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
6365
// vfpdef: NAME
6466

65-
DEF_RULE_NC(decorator, and(4), tok(OP_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
67+
DEF_RULE_NC(decorator, and(3), tok(OP_AT), rule(decorator_arg), tok(NEWLINE))
68+
DEF_RULE_NC(decorator_arg, or(2), rule(built_in_decorator), rule(namedexpr_test))
69+
DEF_RULE_NC(built_in_decorator, and(3), tok(NAME), tok(DEL_PERIOD), tok(NAME))
6670
DEF_RULE_NC(decorators, one_or_more, rule(decorator))
6771
DEF_RULE(decorated, c(decorated), and_ident(2), rule(decorators), rule(decorated_body))
6872
#if MICROPY_PY_ASYNC_AWAIT

‎py/parse.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,11 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
980980
mp_token_kind_t tok_kind = rule_arg[i] & RULE_ARG_ARG_MASK;
981981
if (lex->tok_kind == tok_kind) {
982982
// matched token
983+
if (rule_id == RULE_built_in_decorator && i == 0 && qstr_from_strn(lex->vstr.buf, lex->vstr.len) != MP_QSTR_micropython) {
984+
// The first NAME of a built-in decorator must be "micropython"
985+
backtrack = true;
986+
goto next_rule;
987+
}
983988
if (tok_kind == MP_TOKEN_NAME) {
984989
push_result_token(&parser, rule_id);
985990
}

‎tests/basics/decorator.py

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,71 @@
1-
# test decorators
1+
### Test decorator syntax.
22

3+
###############################################################################
4+
### Test for syntax errors.
5+
###############################################################################
6+
def test_for_syntax_error(code):
7+
try:
8+
exec(code)
9+
except SyntaxError:
10+
print("SyntaxError")
11+
12+
decs = [
13+
"x,",
14+
"x, y",
15+
"x = y",
16+
"pass",
17+
"import sys",
18+
"@dec"
19+
]
20+
21+
for dec in decs:
22+
code = "@{}\ndef foo(): pass".format(dec)
23+
test_for_syntax_error(code)
24+
25+
###############################################################################
26+
### Test for type errors (previously syntax errors before Python 3.9).
27+
###############################################################################
28+
def test_for_type_error(code):
29+
try:
30+
exec(code)
31+
except TypeError:
32+
print("TypeError")
33+
34+
decs = [
35+
"[1, 2][-1]",
36+
"(1, 2)",
37+
"True",
38+
"...",
39+
"None"
40+
]
41+
42+
for dec in decs:
43+
code = "@{}\ndef foo(): pass".format(dec)
44+
test_for_type_error(code)
45+
46+
###############################################################################
47+
### Test for name errors. Tests added to make sure the compiler does not
48+
### confuse anything as a built-in decorator.
49+
###############################################################################
50+
def test_for_name_error(code):
51+
try:
52+
exec(code)
53+
except NameError:
54+
print("NameError")
55+
56+
decs = [
57+
"some",
58+
"some.dotted",
59+
"some.dotted.name"
60+
]
61+
62+
for dec in decs:
63+
code = "@{}\ndef foo(): pass".format(dec)
64+
test_for_name_error(code)
65+
66+
###############################################################################
67+
### Test valid decorators.
68+
###############################################################################
369
def dec(f):
470
print('dec')
571
return f

‎tests/basics/decorator.py.exp

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
SyntaxError
2+
SyntaxError
3+
SyntaxError
4+
SyntaxError
5+
SyntaxError
6+
SyntaxError
7+
TypeError
8+
TypeError
9+
TypeError
10+
TypeError
11+
TypeError
12+
NameError
13+
NameError
14+
NameError
15+
dec
16+
dec_arg
17+
dec

‎tests/cmdline/cmd_parsetree.py.exp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,11 @@
3232
id(h)
3333
[ 13] \(rule\|atom_expr_normal\)(44) (n=2)
3434
[ 13] literal const(\.\+)
35-
[ 13] \(rule\|atom_expr_trailers\)(142) (n=2)
35+
[ 13] \(rule\|atom_expr_trailers\)(144) (n=2)
3636
[ 13] \(rule\|trailer_period\)(50) (n=1)
3737
id(format)
3838
[ 13] \(rule\|trailer_paren\)(48) (n=1)
39-
[ 13] \(rule\|arglist\)(164) (n=1)
39+
[ 13] \(rule\|arglist\)(166) (n=1)
4040
id(b)
4141
----------------
4242
File cmdline/cmd_parsetree.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ 62 bytes)

0 commit comments

Comments
 (0)