From ba4ba82d4f2772e01d9832572cbbfca81475874e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 6 Jan 2019 21:16:27 +0000 Subject: [PATCH 01/36] Some initial infra --- Include/node.h | 4 +++- Modules/parsermodule.c | 2 +- Parser/node.c | 6 +++++- Parser/parser.c | 15 ++++++++------- Parser/parser.h | 3 ++- Parser/parsetok.c | 12 ++++++++++-- Python/ast.c | 1 + 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Include/node.h b/Include/node.h index 40596dfecf1807..2962bb1652175c 100644 --- a/Include/node.h +++ b/Include/node.h @@ -14,11 +14,13 @@ typedef struct _node { int n_col_offset; int n_nchildren; struct _node *n_child; + int n_end_lineno; + int n_end_col_offset; } node; PyAPI_FUNC(node *) PyNode_New(int type); PyAPI_FUNC(int) PyNode_AddChild(node *n, int type, - char *str, int lineno, int col_offset); + char *str, int lineno, int col_offset, int end_col_offset); PyAPI_FUNC(void) PyNode_Free(node *n); #ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 8f88657c00b678..d6ef7b230b16ef 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -920,7 +920,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_DECREF(elem); return NULL; } - err = PyNode_AddChild(root, type, strn, *line_num, 0); + err = PyNode_AddChild(root, type, strn, *line_num, 0, -1); if (err == E_NOMEM) { Py_DECREF(elem); PyObject_FREE(strn); diff --git a/Parser/node.c b/Parser/node.c index 240d29057c4ef1..6533543cc9fada 100644 --- a/Parser/node.c +++ b/Parser/node.c @@ -13,6 +13,8 @@ PyNode_New(int type) n->n_type = type; n->n_str = NULL; n->n_lineno = 0; + n->n_end_lineno = 0; + n->n_end_col_offset = -1; n->n_nchildren = 0; n->n_child = NULL; return n; @@ -76,7 +78,7 @@ fancy_roundup(int n) int -PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset) +PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset, int end_col_offset) { const int nch = n1->n_nchildren; int current_capacity; @@ -107,6 +109,8 @@ PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset) n->n_str = str; n->n_lineno = lineno; n->n_col_offset = col_offset; + n->n_end_lineno = lineno; // this and below are correct only for terminals. + n->n_end_col_offset = end_col_offset; n->n_nchildren = 0; n->n_child = NULL; return 0; diff --git a/Parser/parser.c b/Parser/parser.c index 41072c478c26df..c559d99b312d09 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -105,11 +105,11 @@ PyParser_Delete(parser_state *ps) /* PARSER STACK OPERATIONS */ static int -shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset) +shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset, int end_col_offset) { int err; assert(!s_empty(s)); - err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset); + err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset, end_col_offset); if (err) return err; s->s_top->s_state = newstate; @@ -117,13 +117,13 @@ shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset) } static int -push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset) +push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset, int end_col_offset) { int err; node *n; n = s->s_top->s_parent; assert(!s_empty(s)); - err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset); + err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset, end_col_offset); if (err) return err; s->s_top->s_state = newstate; @@ -225,7 +225,8 @@ future_hack(parser_state *ps) int PyParser_AddToken(parser_state *ps, int type, char *str, - int lineno, int col_offset, int *expected_ret) + int lineno, int col_offset, int end_col_offset, + int *expected_ret) { int ilabel; int err; @@ -257,7 +258,7 @@ PyParser_AddToken(parser_state *ps, int type, char *str, dfa *d1 = PyGrammar_FindDFA( ps->p_grammar, nt); if ((err = push(&ps->p_stack, nt, d1, - arrow, lineno, col_offset)) > 0) { + arrow, lineno, col_offset, end_col_offset)) > 0) { D(printf(" MemError: push\n")); return err; } @@ -267,7 +268,7 @@ PyParser_AddToken(parser_state *ps, int type, char *str, /* Shift the token */ if ((err = shift(&ps->p_stack, type, str, - x, lineno, col_offset)) > 0) { + x, lineno, col_offset, end_col_offset)) > 0) { D(printf(" MemError: shift.\n")); return err; } diff --git a/Parser/parser.h b/Parser/parser.h index 39df9487285c84..373d9e417832d9 100644 --- a/Parser/parser.h +++ b/Parser/parser.h @@ -32,7 +32,8 @@ typedef struct { parser_state *PyParser_New(grammar *g, int start); void PyParser_Delete(parser_state *ps); -int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset, +int PyParser_AddToken(parser_state *ps, int type, char *str, + int lineno, int col_offset, int end_col_offset, int *expected_ret); void PyGrammar_AddAccelerators(grammar *g); diff --git a/Parser/parsetok.c b/Parser/parsetok.c index fc878d89d5630b..c8dbf0ab1dc167 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -187,7 +187,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, parser_state *ps; node *n; int started = 0; - int col_offset; + int col_offset, end_col_offset; if ((ps = PyParser_New(g, start)) == NULL) { err_ret->error = E_NOMEM; @@ -261,9 +261,17 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, col_offset = -1; } + if (b != NULL && b >= tok->line_start) { + end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start, + intptr_t, int); + } + else { + end_col_offset = -1; + } + if ((err_ret->error = PyParser_AddToken(ps, (int)type, str, - tok->lineno, col_offset, + tok->lineno, col_offset, end_col_offset, &(err_ret->expected))) != E_OK) { if (err_ret->error != E_DONE) { PyObject_FREE(str); diff --git a/Python/ast.c b/Python/ast.c index 8a305a80ffac47..e62ca0e59fcaf9 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2303,6 +2303,7 @@ ast_for_binop(struct compiling *c, const node *n) if (!newoperator) return NULL; + printf("%d:%d - %d:%d\n", LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset); result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena); if (!result) From 3e343e3a57006bacd02a989e1b20fc107f21105e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 6 Jan 2019 21:29:26 +0000 Subject: [PATCH 02/36] Regenerate nodes --- Include/Python-ast.h | 273 +++++++++++--------- Parser/Python.asdl | 8 +- Python/Python-ast.c | 578 +++++++++++++++++++++++++++++++++---------- Python/ast.c | 2 + 4 files changed, 619 insertions(+), 242 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 1a2b8297810c84..9d290966cfd89e 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -210,6 +210,8 @@ struct _stmt { } v; int lineno; int col_offset; + int end_lineno; + int end_col_offset; }; enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, @@ -353,6 +355,8 @@ struct _expr { } v; int lineno; int col_offset; + int end_lineno; + int end_col_offset; }; enum _slice_kind {Slice_kind=1, ExtSlice_kind=2, Index_kind=3}; @@ -396,6 +400,8 @@ struct _excepthandler { } v; int lineno; int col_offset; + int end_lineno; + int end_col_offset; }; struct _arguments { @@ -412,6 +418,8 @@ struct _arg { expr_ty annotation; int lineno; int col_offset; + int end_lineno; + int end_col_offset; }; struct _keyword { @@ -438,152 +446,188 @@ mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena); mod_ty _Py_Expression(expr_ty body, PyArena *arena); #define Suite(a0, a1) _Py_Suite(a0, a1) mod_ty _Py_Suite(asdl_seq * body, PyArena *arena); -#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) +#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * decorator_list, expr_ty returns, int lineno, - int col_offset, PyArena *arena); -#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * decorator_list, expr_ty returns, - int lineno, int col_offset, PyArena *arena); -#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq * body, asdl_seq * decorator_list, int lineno, - int col_offset, PyArena *arena); -#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3) -stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena); -#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3) -stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, PyArena - *arena); -#define Assign(a0, a1, a2, a3, a4) _Py_Assign(a0, a1, a2, a3, a4) + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +#define Return(a0, a1, a2, a3, a4, a5) _Py_Return(a0, a1, a2, a3, a4, a5) +stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +#define Delete(a0, a1, a2, a3, a4, a5) _Py_Delete(a0, a1, a2, a3, a4, a5) +stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Assign(a0, a1, a2, a3, a4, a5, a6) _Py_Assign(a0, a1, a2, a3, a4, a5, a6) stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, int lineno, int - col_offset, PyArena *arena); -#define AugAssign(a0, a1, a2, a3, a4, a5) _Py_AugAssign(a0, a1, a2, a3, a4, a5) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int - lineno, int col_offset, PyArena *arena); -#define AnnAssign(a0, a1, a2, a3, a4, a5, a6) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6) + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) stmt_ty _Py_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int - simple, int lineno, int col_offset, PyArena *arena); -#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6) + simple, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8) stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * - orelse, int lineno, int col_offset, PyArena *arena); -#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6) + orelse, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * - orelse, int lineno, int col_offset, PyArena *arena); -#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5) + orelse, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define While(a0, a1, a2, a3, a4, a5, a6, a7) _Py_While(a0, a1, a2, a3, a4, a5, a6, a7) stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, - int col_offset, PyArena *arena); -#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5) + int col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define If(a0, a1, a2, a3, a4, a5, a6, a7) _Py_If(a0, a1, a2, a3, a4, a5, a6, a7) stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, - int col_offset, PyArena *arena); -#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4) + int col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define With(a0, a1, a2, a3, a4, a5, a6) _Py_With(a0, a1, a2, a3, a4, a5, a6) stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, - PyArena *arena); -#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4) + int end_lineno, int end_col_offset, PyArena *arena); +#define AsyncWith(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6) stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int - col_offset, PyArena *arena); -#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4) -stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, - PyArena *arena); -#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define Raise(a0, a1, a2, a3, a4, a5, a6) _Py_Raise(a0, a1, a2, a3, a4, a5, a6) +stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, - asdl_seq * finalbody, int lineno, int col_offset, PyArena - *arena); -#define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4) -stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, - PyArena *arena); -#define Import(a0, a1, a2, a3) _Py_Import(a0, a1, a2, a3) -stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, PyArena - *arena); -#define ImportFrom(a0, a1, a2, a3, a4, a5) _Py_ImportFrom(a0, a1, a2, a3, a4, a5) + asdl_seq * finalbody, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Assert(a0, a1, a2, a3, a4, a5, a6) _Py_Assert(a0, a1, a2, a3, a4, a5, a6) +stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Import(a0, a1, a2, a3, a4, a5) _Py_Import(a0, a1, a2, a3, a4, a5) +stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int - lineno, int col_offset, PyArena *arena); -#define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3) -stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena - *arena); -#define Nonlocal(a0, a1, a2, a3) _Py_Nonlocal(a0, a1, a2, a3) -stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena - *arena); -#define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3) -stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena); -#define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2) -stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena); -#define Break(a0, a1, a2) _Py_Break(a0, a1, a2) -stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena); -#define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2) -stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena); -#define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4) + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Global(a0, a1, a2, a3, a4, a5) _Py_Global(a0, a1, a2, a3, a4, a5) +stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Nonlocal(a0, a1, a2, a3, a4, a5) _Py_Nonlocal(a0, a1, a2, a3, a4, a5) +stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Expr(a0, a1, a2, a3, a4, a5) _Py_Expr(a0, a1, a2, a3, a4, a5) +stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Pass(a0, a1, a2, a3, a4) _Py_Pass(a0, a1, a2, a3, a4) +stmt_ty _Py_Pass(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Break(a0, a1, a2, a3, a4) _Py_Break(a0, a1, a2, a3, a4) +stmt_ty _Py_Break(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Continue(a0, a1, a2, a3, a4) _Py_Continue(a0, a1, a2, a3, a4) +stmt_ty _Py_Continue(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define BoolOp(a0, a1, a2, a3, a4, a5, a6) _Py_BoolOp(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, - PyArena *arena); -#define BinOp(a0, a1, a2, a3, a4, a5) _Py_BinOp(a0, a1, a2, a3, a4, a5) + int end_lineno, int end_col_offset, PyArena *arena); +#define BinOp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_BinOp(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int - col_offset, PyArena *arena); -#define UnaryOp(a0, a1, a2, a3, a4) _Py_UnaryOp(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define UnaryOp(a0, a1, a2, a3, a4, a5, a6) _Py_UnaryOp(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, - PyArena *arena); -#define Lambda(a0, a1, a2, a3, a4) _Py_Lambda(a0, a1, a2, a3, a4) + int end_lineno, int end_col_offset, PyArena *arena); +#define Lambda(a0, a1, a2, a3, a4, a5, a6) _Py_Lambda(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, - PyArena *arena); -#define IfExp(a0, a1, a2, a3, a4, a5) _Py_IfExp(a0, a1, a2, a3, a4, a5) + int end_lineno, int end_col_offset, PyArena *arena); +#define IfExp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_IfExp(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int - col_offset, PyArena *arena); -#define Dict(a0, a1, a2, a3, a4) _Py_Dict(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define Dict(a0, a1, a2, a3, a4, a5, a6) _Py_Dict(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_Dict(asdl_seq * keys, asdl_seq * values, int lineno, int - col_offset, PyArena *arena); -#define Set(a0, a1, a2, a3) _Py_Set(a0, a1, a2, a3) -expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena); -#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define Set(a0, a1, a2, a3, a4, a5) _Py_Set(a0, a1, a2, a3, a4, a5) +expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +#define ListComp(a0, a1, a2, a3, a4, a5, a6) _Py_ListComp(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int - col_offset, PyArena *arena); -#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define SetComp(a0, a1, a2, a3, a4, a5, a6) _Py_SetComp(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int - col_offset, PyArena *arena); -#define DictComp(a0, a1, a2, a3, a4, a5) _Py_DictComp(a0, a1, a2, a3, a4, a5) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define DictComp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_DictComp(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int - lineno, int col_offset, PyArena *arena); -#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4) + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define GeneratorExp(a0, a1, a2, a3, a4, a5, a6) _Py_GeneratorExp(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int - col_offset, PyArena *arena); -#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3) -expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena); -#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3) -expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena); -#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3) -expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena - *arena); -#define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5) + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +#define Await(a0, a1, a2, a3, a4, a5) _Py_Await(a0, a1, a2, a3, a4, a5) +expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +#define Yield(a0, a1, a2, a3, a4, a5) _Py_Yield(a0, a1, a2, a3, a4, a5) +expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +#define YieldFrom(a0, a1, a2, a3, a4, a5) _Py_YieldFrom(a0, a1, a2, a3, a4, a5) +expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Compare(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Compare(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, - int lineno, int col_offset, PyArena *arena); -#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5) + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int - lineno, int col_offset, PyArena *arena); -#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5) + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +#define FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, - int lineno, int col_offset, PyArena *arena); -#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3) -expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena - *arena); -#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3) -expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena - *arena); -#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define JoinedStr(a0, a1, a2, a3, a4, a5) _Py_JoinedStr(a0, a1, a2, a3, a4, a5) +expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Constant(a0, a1, a2, a3, a4, a5) _Py_Constant(a0, a1, a2, a3, a4, a5) +expr_ty _Py_Constant(constant value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define Attribute(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Attribute(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int - lineno, int col_offset, PyArena *arena); -#define Subscript(a0, a1, a2, a3, a4, a5) _Py_Subscript(a0, a1, a2, a3, a4, a5) + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Subscript(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Subscript(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int - lineno, int col_offset, PyArena *arena); -#define Starred(a0, a1, a2, a3, a4) _Py_Starred(a0, a1, a2, a3, a4) + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +#define Starred(a0, a1, a2, a3, a4, a5, a6) _Py_Starred(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_Starred(expr_ty value, expr_context_ty ctx, int lineno, int - col_offset, PyArena *arena); -#define Name(a0, a1, a2, a3, a4) _Py_Name(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define Name(a0, a1, a2, a3, a4, a5, a6) _Py_Name(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_Name(identifier id, expr_context_ty ctx, int lineno, int - col_offset, PyArena *arena); -#define List(a0, a1, a2, a3, a4) _Py_List(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define List(a0, a1, a2, a3, a4, a5, a6) _Py_List(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_List(asdl_seq * elts, expr_context_ty ctx, int lineno, int - col_offset, PyArena *arena); -#define Tuple(a0, a1, a2, a3, a4) _Py_Tuple(a0, a1, a2, a3, a4) + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +#define Tuple(a0, a1, a2, a3, a4, a5, a6) _Py_Tuple(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int - col_offset, PyArena *arena); + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); #define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3) slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); #define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1) @@ -593,17 +637,18 @@ slice_ty _Py_Index(expr_ty value, PyArena *arena); #define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4) comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async, PyArena *arena); -#define ExceptHandler(a0, a1, a2, a3, a4, a5) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5) +#define ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq * - body, int lineno, int col_offset, PyArena + body, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); #define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6) arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg, asdl_seq * defaults, PyArena *arena); -#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4) +#define arg(a0, a1, a2, a3, a4, a5, a6) _Py_arg(a0, a1, a2, a3, a4, a5, a6) arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, - PyArena *arena); + int end_lineno, int end_col_offset, PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) diff --git a/Parser/Python.asdl b/Parser/Python.asdl index eee982be1c9545..cedf37a2d9f9ee 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -50,7 +50,7 @@ module Python -- XXX Jython will be different -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) @@ -85,7 +85,7 @@ module Python | Tuple(expr* elts, expr_context ctx) -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) expr_context = Load | Store | Del | AugLoad | AugStore | Param @@ -105,13 +105,13 @@ module Python comprehension = (expr target, expr iter, expr* ifs, int is_async) excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults) arg = (identifier arg, expr? annotation) - attributes (int lineno, int col_offset) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) -- keyword arguments supplied to call (NULL identifier for **kwargs) keyword = (identifier? arg, expr value) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index bbe8e69fdd504c..e6c5bfe9b29e00 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -28,9 +28,13 @@ static char *Suite_fields[]={ static PyTypeObject *stmt_type; _Py_IDENTIFIER(lineno); _Py_IDENTIFIER(col_offset); +_Py_IDENTIFIER(end_lineno); +_Py_IDENTIFIER(end_col_offset); static char *stmt_attributes[] = { "lineno", "col_offset", + "end_lineno", + "end_col_offset", }; static PyObject* ast2obj_stmt(void*); static PyTypeObject *FunctionDef_type; @@ -189,6 +193,8 @@ static PyTypeObject *expr_type; static char *expr_attributes[] = { "lineno", "col_offset", + "end_lineno", + "end_col_offset", }; static PyObject* ast2obj_expr(void*); static PyTypeObject *BoolOp_type; @@ -427,6 +433,8 @@ static PyTypeObject *excepthandler_type; static char *excepthandler_attributes[] = { "lineno", "col_offset", + "end_lineno", + "end_col_offset", }; static PyObject* ast2obj_excepthandler(void*); static PyTypeObject *ExceptHandler_type; @@ -456,6 +464,8 @@ static PyObject* ast2obj_arg(void*); static char *arg_attributes[] = { "lineno", "col_offset", + "end_lineno", + "end_col_offset", }; _Py_IDENTIFIER(arg); static char *arg_fields[]={ @@ -804,7 +814,7 @@ static int init_types(void) if (!Suite_type) return 0; stmt_type = make_type("stmt", &AST_type, NULL, 0); if (!stmt_type) return 0; - if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0; + if (!add_attributes(stmt_type, stmt_attributes, 4)) return 0; FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields, 5); if (!FunctionDef_type) return 0; @@ -859,7 +869,7 @@ static int init_types(void) if (!Continue_type) return 0; expr_type = make_type("expr", &AST_type, NULL, 0); if (!expr_type) return 0; - if (!add_attributes(expr_type, expr_attributes, 2)) return 0; + if (!add_attributes(expr_type, expr_attributes, 4)) return 0; BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2); if (!BoolOp_type) return 0; BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3); @@ -1082,7 +1092,7 @@ static int init_types(void) if (!add_attributes(comprehension_type, NULL, 0)) return 0; excepthandler_type = make_type("excepthandler", &AST_type, NULL, 0); if (!excepthandler_type) return 0; - if (!add_attributes(excepthandler_type, excepthandler_attributes, 2)) + if (!add_attributes(excepthandler_type, excepthandler_attributes, 4)) return 0; ExceptHandler_type = make_type("ExceptHandler", excepthandler_type, ExceptHandler_fields, 3); @@ -1092,7 +1102,7 @@ static int init_types(void) if (!add_attributes(arguments_type, NULL, 0)) return 0; arg_type = make_type("arg", &AST_type, arg_fields, 2); if (!arg_type) return 0; - if (!add_attributes(arg_type, arg_attributes, 2)) return 0; + if (!add_attributes(arg_type, arg_attributes, 4)) return 0; keyword_type = make_type("keyword", &AST_type, keyword_fields, 2); if (!keyword_type) return 0; if (!add_attributes(keyword_type, NULL, 0)) return 0; @@ -1181,8 +1191,8 @@ Suite(asdl_seq * body, PyArena *arena) stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * - decorator_list, expr_ty returns, int lineno, int col_offset, - PyArena *arena) + decorator_list, expr_ty returns, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1206,13 +1216,15 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * p->v.FunctionDef.returns = returns; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * decorator_list, expr_ty returns, int lineno, int col_offset, - PyArena *arena) + int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1236,13 +1248,15 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq p->v.AsyncFunctionDef.returns = returns; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq * - body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena - *arena) + body, asdl_seq * decorator_list, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1261,11 +1275,14 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq * p->v.ClassDef.decorator_list = decorator_list; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Return(expr_ty value, int lineno, int col_offset, PyArena *arena) +Return(expr_ty value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1275,11 +1292,14 @@ Return(expr_ty value, int lineno, int col_offset, PyArena *arena) p->v.Return.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena) +Delete(asdl_seq * targets, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1289,12 +1309,14 @@ Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena) p->v.Delete.targets = targets; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena - *arena) +Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!value) { @@ -1310,12 +1332,14 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena p->v.Assign.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!target) { @@ -1342,12 +1366,15 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int p->v.AugAssign.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int - lineno, int col_offset, PyArena *arena) + lineno, int col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { stmt_ty p; if (!target) { @@ -1370,12 +1397,14 @@ AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int p->v.AnnAssign.simple = simple; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int - lineno, int col_offset, PyArena *arena) + lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!target) { @@ -1398,12 +1427,15 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int p->v.For.orelse = orelse; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int - lineno, int col_offset, PyArena *arena) + lineno, int col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { stmt_ty p; if (!target) { @@ -1426,12 +1458,14 @@ AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int p->v.AsyncFor.orelse = orelse; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!test) { @@ -1448,12 +1482,14 @@ While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int p->v.While.orelse = orelse; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!test) { @@ -1470,12 +1506,14 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int p->v.If.orelse = orelse; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena - *arena) +With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1486,12 +1524,14 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena p->v.With.body = body; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, - PyArena *arena) +AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1502,11 +1542,14 @@ AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, p->v.AsyncWith.body = body; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena) +Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1517,12 +1560,15 @@ Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena) p->v.Raise.cause = cause; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq * - finalbody, int lineno, int col_offset, PyArena *arena) + finalbody, int lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1535,11 +1581,14 @@ Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq * p->v.Try.finalbody = finalbody; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena) +Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena) { stmt_ty p; if (!test) { @@ -1555,11 +1604,14 @@ Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena) p->v.Assert.msg = msg; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena) +Import(asdl_seq * names, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1569,12 +1621,14 @@ Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena) p->v.Import.names = names; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1586,11 +1640,14 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int p->v.ImportFrom.level = level; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena) +Global(asdl_seq * names, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1600,11 +1657,14 @@ Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena) p->v.Global.names = names; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena) +Nonlocal(asdl_seq * names, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1614,11 +1674,14 @@ Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena) p->v.Nonlocal.names = names; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Expr(expr_ty value, int lineno, int col_offset, PyArena *arena) +Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; if (!value) { @@ -1633,11 +1696,14 @@ Expr(expr_ty value, int lineno, int col_offset, PyArena *arena) p->v.Expr.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Pass(int lineno, int col_offset, PyArena *arena) +Pass(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1646,11 +1712,14 @@ Pass(int lineno, int col_offset, PyArena *arena) p->kind = Pass_kind; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Break(int lineno, int col_offset, PyArena *arena) +Break(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1659,11 +1728,14 @@ Break(int lineno, int col_offset, PyArena *arena) p->kind = Break_kind; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } stmt_ty -Continue(int lineno, int col_offset, PyArena *arena) +Continue(int lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1672,12 +1744,14 @@ Continue(int lineno, int col_offset, PyArena *arena) p->kind = Continue_kind; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena - *arena) +BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!op) { @@ -1693,12 +1767,14 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena p->v.BoolOp.values = values; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset, - PyArena *arena) + int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!left) { @@ -1725,12 +1801,14 @@ BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset, p->v.BinOp.right = right; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena - *arena) +UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!op) { @@ -1751,12 +1829,14 @@ UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena p->v.UnaryOp.operand = operand; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena - *arena) +Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!args) { @@ -1777,12 +1857,14 @@ Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena p->v.Lambda.body = body; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset, - PyArena *arena) + int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!test) { @@ -1809,12 +1891,14 @@ IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset, p->v.IfExp.orelse = orelse; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena - *arena) +Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1825,11 +1909,14 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena p->v.Dict.values = values; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena) +Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1839,12 +1926,14 @@ Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena) p->v.Set.elts = elts; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, - PyArena *arena) +ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!elt) { @@ -1860,12 +1949,14 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, p->v.ListComp.generators = generators; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena - *arena) +SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!elt) { @@ -1881,12 +1972,14 @@ SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena p->v.SetComp.generators = generators; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!key) { @@ -1908,12 +2001,14 @@ DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int p->v.DictComp.generators = generators; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, - PyArena *arena) + int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!elt) { @@ -1929,11 +2024,14 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, p->v.GeneratorExp.generators = generators; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Await(expr_ty value, int lineno, int col_offset, PyArena *arena) +Await(expr_ty value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -1948,11 +2046,14 @@ Await(expr_ty value, int lineno, int col_offset, PyArena *arena) p->v.Await.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Yield(expr_ty value, int lineno, int col_offset, PyArena *arena) +Yield(expr_ty value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1962,11 +2063,14 @@ Yield(expr_ty value, int lineno, int col_offset, PyArena *arena) p->v.Yield.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena) +YieldFrom(expr_ty value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -1981,12 +2085,14 @@ YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena) p->v.YieldFrom.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, - int col_offset, PyArena *arena) + int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!left) { @@ -2003,12 +2109,14 @@ Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, p->v.Compare.comparators = comparators; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!func) { @@ -2025,12 +2133,15 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int p->v.Call.keywords = keywords; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno, - int col_offset, PyArena *arena) + int col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { expr_ty p; if (!value) { @@ -2047,11 +2158,14 @@ FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno, p->v.FormattedValue.format_spec = format_spec; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena) +JoinedStr(asdl_seq * values, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -2061,11 +2175,14 @@ JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena) p->v.JoinedStr.values = values; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Constant(constant value, int lineno, int col_offset, PyArena *arena) +Constant(constant value, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -2080,12 +2197,14 @@ Constant(constant value, int lineno, int col_offset, PyArena *arena) p->v.Constant.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -2112,12 +2231,14 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int p->v.Attribute.ctx = ctx; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -2144,12 +2265,14 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int p->v.Subscript.ctx = ctx; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena - *arena) +Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -2170,12 +2293,14 @@ Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena p->v.Starred.ctx = ctx; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena - *arena) +Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!id) { @@ -2196,12 +2321,14 @@ Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena p->v.Name.ctx = ctx; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena - *arena) +List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!ctx) { @@ -2217,12 +2344,14 @@ List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena p->v.List.ctx = ctx; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } expr_ty -Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena - *arena) +Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!ctx) { @@ -2238,6 +2367,8 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena p->v.Tuple.ctx = ctx; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } @@ -2311,7 +2442,7 @@ comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async, excepthandler_ty ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int - col_offset, PyArena *arena) + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { excepthandler_ty p; p = (excepthandler_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -2323,6 +2454,8 @@ ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int p->v.ExceptHandler.body = body; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } @@ -2344,8 +2477,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * } arg_ty -arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena - *arena) +arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { arg_ty p; if (!arg) { @@ -2360,6 +2493,8 @@ arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena p->annotation = annotation; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } @@ -2886,6 +3021,16 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); + value = ast2obj_int(o->end_lineno); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->end_col_offset); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -3281,6 +3426,16 @@ ast2obj_expr(void* _o) if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); + value = ast2obj_int(o->end_lineno); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->end_col_offset); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -3571,6 +3726,16 @@ ast2obj_excepthandler(void* _o) if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); + value = ast2obj_int(o->end_lineno); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->end_col_offset); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -3657,6 +3822,16 @@ ast2obj_arg(void* _o) if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); + value = ast2obj_int(o->end_lineno); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->end_col_offset); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -3922,6 +4097,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyObject *tmp = NULL; int lineno; int col_offset; + int end_lineno; + int end_col_offset; if (obj == Py_None) { *out = NULL; @@ -3953,6 +4130,32 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } + if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_lineno = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_col_offset = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionDef_type); if (isinstance == -1) { return 1; @@ -4064,7 +4267,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_CLEAR(tmp); } *out = FunctionDef(name, args, body, decorator_list, returns, lineno, - col_offset, arena); + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4179,7 +4382,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_CLEAR(tmp); } *out = AsyncFunctionDef(name, args, body, decorator_list, returns, - lineno, col_offset, arena); + lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -4328,7 +4532,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_CLEAR(tmp); } *out = ClassDef(name, bases, keywords, body, decorator_list, lineno, - col_offset, arena); + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4352,7 +4556,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Return(value, lineno, col_offset, arena); + *out = Return(value, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -4393,7 +4598,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Delete(targets, lineno, col_offset, arena); + *out = Delete(targets, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -4448,7 +4654,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Assign(targets, value, lineno, col_offset, arena); + *out = Assign(targets, value, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4500,7 +4707,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = AugAssign(target, op, value, lineno, col_offset, arena); + *out = AugAssign(target, op, value, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4567,7 +4775,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_CLEAR(tmp); } *out = AnnAssign(target, annotation, value, simple, lineno, col_offset, - arena); + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4667,7 +4875,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = For(target, iter, body, orelse, lineno, col_offset, arena); + *out = For(target, iter, body, orelse, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4767,7 +4976,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, arena); + *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4853,7 +5063,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = While(test, body, orelse, lineno, col_offset, arena); + *out = While(test, body, orelse, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4939,7 +5150,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = If(test, body, orelse, lineno, col_offset, arena); + *out = If(test, body, orelse, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5011,7 +5223,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = With(items, body, lineno, col_offset, arena); + *out = With(items, body, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5083,7 +5296,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = AsyncWith(items, body, lineno, col_offset, arena); + *out = AsyncWith(items, body, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5121,7 +5335,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Raise(exc, cause, lineno, col_offset, arena); + *out = Raise(exc, cause, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5256,7 +5471,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_CLEAR(tmp); } *out = Try(body, handlers, orelse, finalbody, lineno, col_offset, - arena); + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5294,7 +5509,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Assert(test, msg, lineno, col_offset, arena); + *out = Assert(test, msg, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5335,7 +5551,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Import(names, lineno, col_offset, arena); + *out = Import(names, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -5404,7 +5621,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = ImportFrom(module, names, level, lineno, col_offset, arena); + *out = ImportFrom(module, names, level, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5445,7 +5663,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Global(names, lineno, col_offset, arena); + *out = Global(names, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -5486,7 +5705,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Nonlocal(names, lineno, col_offset, arena); + *out = Nonlocal(names, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -5510,7 +5730,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Expr(value, lineno, col_offset, arena); + *out = Expr(value, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -5520,7 +5741,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } if (isinstance) { - *out = Pass(lineno, col_offset, arena); + *out = Pass(lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5530,7 +5751,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } if (isinstance) { - *out = Break(lineno, col_offset, arena); + *out = Break(lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5540,7 +5761,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } if (isinstance) { - *out = Continue(lineno, col_offset, arena); + *out = Continue(lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5559,6 +5780,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyObject *tmp = NULL; int lineno; int col_offset; + int end_lineno; + int end_col_offset; if (obj == Py_None) { *out = NULL; @@ -5590,6 +5813,32 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } + if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_lineno = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_col_offset = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } isinstance = PyObject_IsInstance(obj, (PyObject*)BoolOp_type); if (isinstance == -1) { return 1; @@ -5641,7 +5890,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = BoolOp(op, values, lineno, col_offset, arena); + *out = BoolOp(op, values, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5693,7 +5943,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = BinOp(left, op, right, lineno, col_offset, arena); + *out = BinOp(left, op, right, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5731,7 +5982,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = UnaryOp(op, operand, lineno, col_offset, arena); + *out = UnaryOp(op, operand, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5769,7 +6021,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Lambda(args, body, lineno, col_offset, arena); + *out = Lambda(args, body, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5821,7 +6074,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = IfExp(test, body, orelse, lineno, col_offset, arena); + *out = IfExp(test, body, orelse, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5893,7 +6147,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Dict(keys, values, lineno, col_offset, arena); + *out = Dict(keys, values, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5934,7 +6189,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Set(elts, lineno, col_offset, arena); + *out = Set(elts, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5989,7 +6244,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = ListComp(elt, generators, lineno, col_offset, arena); + *out = ListComp(elt, generators, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6044,7 +6300,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = SetComp(elt, generators, lineno, col_offset, arena); + *out = SetComp(elt, generators, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6113,7 +6370,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = DictComp(key, value, generators, lineno, col_offset, arena); + *out = DictComp(key, value, generators, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6168,7 +6426,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = GeneratorExp(elt, generators, lineno, col_offset, arena); + *out = GeneratorExp(elt, generators, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6192,7 +6451,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Await(value, lineno, col_offset, arena); + *out = Await(value, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -6216,7 +6476,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Yield(value, lineno, col_offset, arena); + *out = Yield(value, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -6240,7 +6501,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = YieldFrom(value, lineno, col_offset, arena); + *out = YieldFrom(value, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -6326,7 +6588,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Compare(left, ops, comparators, lineno, col_offset, arena); + *out = Compare(left, ops, comparators, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6412,7 +6675,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Call(func, args, keywords, lineno, col_offset, arena); + *out = Call(func, args, keywords, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6465,7 +6729,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) Py_CLEAR(tmp); } *out = FormattedValue(value, conversion, format_spec, lineno, - col_offset, arena); + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6506,7 +6770,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = JoinedStr(values, lineno, col_offset, arena); + *out = JoinedStr(values, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6530,7 +6795,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Constant(value, lineno, col_offset, arena); + *out = Constant(value, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -6582,7 +6848,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Attribute(value, attr, ctx, lineno, col_offset, arena); + *out = Attribute(value, attr, ctx, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6634,7 +6901,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Subscript(value, slice, ctx, lineno, col_offset, arena); + *out = Subscript(value, slice, ctx, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6672,7 +6940,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Starred(value, ctx, lineno, col_offset, arena); + *out = Starred(value, ctx, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6710,7 +6979,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Name(id, ctx, lineno, col_offset, arena); + *out = Name(id, ctx, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -6765,7 +7035,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = List(elts, ctx, lineno, col_offset, arena); + *out = List(elts, ctx, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -6820,7 +7091,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Tuple(elts, ctx, lineno, col_offset, arena); + *out = Tuple(elts, ctx, lineno, col_offset, end_lineno, end_col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -7389,6 +7661,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) PyObject *tmp = NULL; int lineno; int col_offset; + int end_lineno; + int end_col_offset; if (obj == Py_None) { *out = NULL; @@ -7420,6 +7694,32 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } + if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_lineno = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_col_offset = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } isinstance = PyObject_IsInstance(obj, (PyObject*)ExceptHandler_type); if (isinstance == -1) { return 1; @@ -7485,7 +7785,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = ExceptHandler(type, name, body, lineno, col_offset, arena); + *out = ExceptHandler(type, name, body, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -7669,6 +7970,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) expr_ty annotation; int lineno; int col_offset; + int end_lineno; + int end_col_offset; if (_PyObject_LookupAttrId(obj, &PyId_arg, &tmp) < 0) { return 1; @@ -7722,7 +8025,34 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = arg(arg, annotation, lineno, col_offset, arena); + if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_lineno = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_col_offset = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = arg(arg, annotation, lineno, col_offset, end_lineno, end_col_offset, + arena); return 0; failed: Py_XDECREF(tmp); diff --git a/Python/ast.c b/Python/ast.c index e62ca0e59fcaf9..496b21967dc1f3 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -938,6 +938,8 @@ copy_location(expr_ty e, const node *n) if (e) { e->lineno = LINENO(n); e->col_offset = n->n_col_offset; + e->end_lineno = n->n_end_lineno; + e->end_col_offset = n->n_end_col_offset; } return e; } From 1684c17c10662ffdae69a60dc48955b93848e165 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 6 Jan 2019 23:21:19 +0000 Subject: [PATCH 03/36] Mindless implementation: known bugs, notably in fstrings --- Include/ast.h | 4 +- Python/ast.c | 203 +++++++++++++++++++++++++++++------------------ Python/ast_opt.c | 2 +- Python/compile.c | 4 +- 4 files changed, 129 insertions(+), 84 deletions(-) diff --git a/Include/ast.h b/Include/ast.h index f1d734852eca79..545706525ab05f 100644 --- a/Include/ast.h +++ b/Include/ast.h @@ -9,12 +9,12 @@ extern "C" { PyAPI_FUNC(int) PyAST_Validate(mod_ty); PyAPI_FUNC(mod_ty) PyAST_FromNode( - const node *n, + node *n, PyCompilerFlags *flags, const char *filename, /* decoded from the filesystem encoding */ PyArena *arena); PyAPI_FUNC(mod_ty) PyAST_FromNodeObject( - const node *n, + node *n, PyCompilerFlags *flags, PyObject *filename, PyArena *arena); diff --git a/Python/ast.c b/Python/ast.c index 496b21967dc1f3..401339cb9ef616 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -739,11 +739,32 @@ num_stmts(const node *n) Py_UNREACHABLE(); } +void +update_node_ends(node *n) +{ + int max_line = 0, max_col = -1; + node *ch; + + if (NCH(n) == 0) { + return; + } + + for (int i = 0; i < NCH(n); i++) { + ch = CHILD(n, i); + update_node_ends(ch); + max_line = ch->n_end_lineno > max_line ? ch->n_end_lineno : max_line; + max_col = ch->n_end_col_offset > max_col ? ch->n_end_col_offset : max_col; + } + n->n_end_lineno = max_line; + n->n_end_col_offset = max_col; +} + + /* Transform the CST rooted at node * to the appropriate AST */ mod_ty -PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, +PyAST_FromNodeObject(node *n, PyCompilerFlags *flags, PyObject *filename, PyArena *arena) { int i, j, k, num; @@ -758,6 +779,8 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, c.c_filename = filename; c.c_normalize = NULL; + update_node_ends(n); + if (TYPE(n) == encoding_decl) n = CHILD(n, 0); @@ -808,6 +831,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, if (!stmts) goto out; asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, arena)); if (!asdl_seq_GET(stmts, 0)) goto out; @@ -854,7 +878,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, } mod_ty -PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename_str, +PyAST_FromNode(node *n, PyCompilerFlags *flags, const char *filename_str, PyArena *arena) { mod_ty mod; @@ -1228,7 +1252,7 @@ ast_for_arg(struct compiling *c, const node *n) return NULL; } - ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena); + ret = arg(name, annotation, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!ret) return NULL; return ret; @@ -1287,6 +1311,7 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, if (forbidden_name(c, argname, ch, 0)) goto error; arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset, + ch->n_end_lineno, ch->n_end_col_offset, c->c_arena); if (!arg) goto error; @@ -1480,16 +1505,18 @@ ast_for_dotted_name(struct compiling *c, const node *n) identifier id; int lineno, col_offset; int i; + node *ch; REQ(n, dotted_name); lineno = LINENO(n); col_offset = n->n_col_offset; - id = NEW_IDENTIFIER(CHILD(n, 0)); + ch = CHILD(n, 0); + id = NEW_IDENTIFIER(ch); if (!id) return NULL; - e = Name(id, Load, lineno, col_offset, c->c_arena); + e = Name(id, Load, lineno, col_offset, ch->n_end_lineno, ch->n_end_col_offset, c->c_arena); if (!e) return NULL; @@ -1497,7 +1524,7 @@ ast_for_dotted_name(struct compiling *c, const node *n) id = NEW_IDENTIFIER(CHILD(n, i)); if (!id) return NULL; - e = Attribute(e, id, Load, lineno, col_offset, c->c_arena); + e = Attribute(e, id, Load, lineno, col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!e) return NULL; } @@ -1526,7 +1553,7 @@ ast_for_decorator(struct compiling *c, const node *n) } else if (NCH(n) == 5) { /* Call with no arguments */ d = Call(name_expr, NULL, NULL, LINENO(n), - n->n_col_offset, c->c_arena); + n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!d) return NULL; name_expr = NULL; @@ -1596,10 +1623,10 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, if (is_async) return AsyncFunctionDef(name, args, body, decorator_seq, returns, - LINENO(n0), n0->n_col_offset, c->c_arena); + LINENO(n0), n0->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else return FunctionDef(name, args, body, decorator_seq, returns, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -1704,7 +1731,7 @@ ast_for_lambdef(struct compiling *c, const node *n) return NULL; } - return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena); + return Lambda(args, expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -1724,6 +1751,7 @@ ast_for_ifexpr(struct compiling *c, const node *n) if (!orelse) return NULL; return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } @@ -1853,7 +1881,7 @@ ast_for_comprehension(struct compiling *c, const node *n) is_async, c->c_arena); else comp = comprehension(Tuple(t, Store, first->lineno, - first->col_offset, c->c_arena), + first->col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena), expression, NULL, is_async, c->c_arena); if (!comp) return NULL; @@ -1918,11 +1946,11 @@ ast_for_itercomp(struct compiling *c, const node *n, int type) return NULL; if (type == COMP_GENEXP) - return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else if (type == COMP_LISTCOMP) - return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + return ListComp(elt, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else if (type == COMP_SETCOMP) - return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + return SetComp(elt, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else /* Should never happen */ return NULL; @@ -1984,7 +2012,7 @@ ast_for_dictcomp(struct compiling *c, const node *n) if (!comps) return NULL; - return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena); + return DictComp(key, value, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2017,7 +2045,7 @@ ast_for_dictdisplay(struct compiling *c, const node *n) } keys->size = j; values->size = j; - return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena); + return Dict(keys, values, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2060,7 +2088,7 @@ ast_for_setdisplay(struct compiling *c, const node *n) return NULL; asdl_seq_SET(elts, i / 2, expression); } - return Set(elts, LINENO(n), n->n_col_offset, c->c_arena); + return Set(elts, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2079,17 +2107,17 @@ ast_for_atom(struct compiling *c, const node *n) size_t len = strlen(s); if (len >= 4 && len <= 5) { if (!strcmp(s, "None")) - return Constant(Py_None, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(Py_None, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!strcmp(s, "True")) - return Constant(Py_True, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(Py_True, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!strcmp(s, "False")) - return Constant(Py_False, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(Py_False, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } name = new_identifier(s, c); if (!name) return NULL; /* All names start in Load context, but may later be changed. */ - return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); + return Name(name, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case STRING: { expr_ty str = parsestrplus(c, n); @@ -2128,15 +2156,15 @@ ast_for_atom(struct compiling *c, const node *n) Py_DECREF(pynum); return NULL; } - return Constant(pynum, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(pynum, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case ELLIPSIS: /* Ellipsis */ - return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); if (TYPE(ch) == RPAR) - return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); + return Tuple(NULL, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); @@ -2156,7 +2184,7 @@ ast_for_atom(struct compiling *c, const node *n) ch = CHILD(n, 1); if (TYPE(ch) == RSQB) - return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); + return List(NULL, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); REQ(ch, testlist_comp); if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { @@ -2164,7 +2192,7 @@ ast_for_atom(struct compiling *c, const node *n) if (!elts) return NULL; - return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena); + return List(elts, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { return copy_location(ast_for_listcomp(c, ch), n); @@ -2178,7 +2206,7 @@ ast_for_atom(struct compiling *c, const node *n) ch = CHILD(n, 1); if (TYPE(ch) == RBRACE) { /* It's an empty dict. */ - return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); + return Dict(NULL, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { int is_dict = (TYPE(CHILD(ch, 0)) == DOUBLESTAR); @@ -2305,8 +2333,8 @@ ast_for_binop(struct compiling *c, const node *n) if (!newoperator) return NULL; - printf("%d:%d - %d:%d\n", LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset); result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, + CHILD(n, 2)->n_end_lineno, CHILD(n, 2)->n_end_col_offset, c->c_arena); if (!result) return NULL; @@ -2326,6 +2354,7 @@ ast_for_binop(struct compiling *c, const node *n) tmp_result = BinOp(result, newoperator, tmp, LINENO(next_oper), next_oper->n_col_offset, + CHILD(n, i * 2 + 2)->n_end_lineno, CHILD(n, i * 2 + 2)->n_end_col_offset, c->c_arena); if (!tmp_result) return NULL; @@ -2345,7 +2374,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (TYPE(CHILD(n, 0)) == LPAR) { if (NCH(n) == 2) return Call(left_expr, NULL, NULL, LINENO(n), - n->n_col_offset, c->c_arena); + n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0)); } @@ -2354,7 +2383,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!attr_id) return NULL; return Attribute(left_expr, attr_id, Load, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { REQ(CHILD(n, 0), LSQB); @@ -2365,6 +2394,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!slc) return NULL; return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { @@ -2390,7 +2420,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) } if (!simple) { return Subscript(left_expr, ExtSlice(slices, c->c_arena), - Load, LINENO(n), n->n_col_offset, c->c_arena); + Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* extract Index values and put them in a Tuple */ elts = _Py_asdl_seq_new(asdl_seq_LEN(slices), c->c_arena); @@ -2401,11 +2431,11 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) assert(slc->kind == Index_kind && slc->v.Index.value); asdl_seq_SET(elts, j, slc->v.Index.value); } - e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena); + e = Tuple(elts, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!e) return NULL; return Subscript(left_expr, Index(e, c->c_arena), - Load, LINENO(n), n->n_col_offset, c->c_arena); + Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } } @@ -2422,12 +2452,15 @@ ast_for_factor(struct compiling *c, const node *n) switch (TYPE(CHILD(n, 0))) { case PLUS: return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case MINUS: return UnaryOp(USub, expression, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case TILDE: return UnaryOp(Invert, expression, LINENO(n), + n->n_end_lineno, n->n_end_col_offset, n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, "unhandled factor: %d", @@ -2455,7 +2488,7 @@ ast_for_atom_expr(struct compiling *c, const node *n) if (nch == 1) return e; if (start && nch == 2) { - return Await(e, LINENO(n), n->n_col_offset, c->c_arena); + return Await(e, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } for (i = start + 1; i < nch; i++) { @@ -2472,7 +2505,7 @@ ast_for_atom_expr(struct compiling *c, const node *n) if (start) { /* there was an 'await' */ - return Await(e, LINENO(n), n->n_col_offset, c->c_arena); + return Await(e, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { return e; @@ -2495,7 +2528,7 @@ ast_for_power(struct compiling *c, const node *n) expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1)); if (!f) return NULL; - e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena); + e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } return e; } @@ -2511,7 +2544,7 @@ ast_for_starred(struct compiling *c, const node *n) return NULL; /* The Load context is changed later. */ - return Starred(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena); + return Starred(tmp, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } @@ -2570,9 +2603,10 @@ ast_for_expr(struct compiling *c, const node *n) } if (!strcmp(STR(CHILD(n, 1)), "and")) return BoolOp(And, seq, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); assert(!strcmp(STR(CHILD(n, 1)), "or")); - return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena); + return BoolOp(Or, seq, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); case not_test: if (NCH(n) == 1) { n = CHILD(n, 0); @@ -2584,6 +2618,7 @@ ast_for_expr(struct compiling *c, const node *n) return NULL; return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case comparison: @@ -2624,6 +2659,7 @@ ast_for_expr(struct compiling *c, const node *n) } return Compare(expression, ops, cmps, LINENO(n), + n->n_end_lineno, n->n_end_col_offset, n->n_col_offset, c->c_arena); } break; @@ -2664,8 +2700,8 @@ ast_for_expr(struct compiling *c, const node *n) return NULL; } if (is_from) - return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena); - return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); + return YieldFrom(exp, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Yield(exp, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case factor: if (NCH(n) == 1) { @@ -2774,6 +2810,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, return NULL; starred = Starred(e, Load, LINENO(chch), chch->n_col_offset, + chch->n_end_lineno, chch->n_end_col_offset, c->c_arena); if (!starred) return NULL; @@ -2865,7 +2902,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, } } - return Call(func, args, keywords, func->lineno, func->col_offset, c->c_arena); + return Call(func, args, keywords, func->lineno, func->col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2888,7 +2925,7 @@ ast_for_testlist(struct compiling *c, const node* n) asdl_seq *tmp = seq_for_testlist(c, n); if (!tmp) return NULL; - return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena); + return Tuple(tmp, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } @@ -2910,7 +2947,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!e) return NULL; - return Expr(e, LINENO(n), n->n_col_offset, c->c_arena); + return Expr(e, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (TYPE(CHILD(n, 1)) == augassign) { expr_ty expr1, expr2; @@ -2948,7 +2985,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!newoperator) return NULL; - return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena); + return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (TYPE(CHILD(n, 1)) == annassign) { expr_ty expr1, expr2, expr3; @@ -3008,7 +3045,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) } if (NCH(ann) == 2) { return AnnAssign(expr1, expr2, NULL, simple, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { ch = CHILD(ann, 3); @@ -3017,7 +3054,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) return NULL; } return AnnAssign(expr1, expr2, expr3, simple, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } else { @@ -3055,7 +3092,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) expression = ast_for_expr(c, value); if (!expression) return NULL; - return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena); + return Assign(targets, expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } @@ -3094,7 +3131,7 @@ ast_for_del_stmt(struct compiling *c, const node *n) expr_list = ast_for_exprlist(c, CHILD(n, 1), Del); if (!expr_list) return NULL; - return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena); + return Delete(expr_list, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3116,27 +3153,27 @@ ast_for_flow_stmt(struct compiling *c, const node *n) ch = CHILD(n, 0); switch (TYPE(ch)) { case break_stmt: - return Break(LINENO(n), n->n_col_offset, c->c_arena); + return Break(LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); case continue_stmt: - return Continue(LINENO(n), n->n_col_offset, c->c_arena); + return Continue(LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); case yield_stmt: { /* will reduce to yield_expr */ expr_ty exp = ast_for_expr(c, CHILD(ch, 0)); if (!exp) return NULL; - return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena); + return Expr(exp, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case return_stmt: if (NCH(ch) == 1) - return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena); + return Return(NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else { expr_ty expression = ast_for_testlist(c, CHILD(ch, 1)); if (!expression) return NULL; - return Return(expression, LINENO(n), n->n_col_offset, c->c_arena); + return Return(expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case raise_stmt: if (NCH(ch) == 1) - return Raise(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); + return Raise(NULL, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else if (NCH(ch) >= 2) { expr_ty cause = NULL; expr_ty expression = ast_for_expr(c, CHILD(ch, 1)); @@ -3147,7 +3184,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n) if (!cause) return NULL; } - return Raise(expression, cause, LINENO(n), n->n_col_offset, c->c_arena); + return Raise(expression, cause, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* fall through */ default: @@ -3308,7 +3345,7 @@ ast_for_import_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(aliases, i / 2, import_alias); } - return Import(aliases, lineno, col_offset, c->c_arena); + return Import(aliases, lineno, col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (TYPE(n) == import_from) { int n_children; @@ -3383,6 +3420,7 @@ ast_for_import_stmt(struct compiling *c, const node *n) if (mod != NULL) modname = mod->name; return ImportFrom(modname, aliases, ndots, lineno, col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3409,7 +3447,7 @@ ast_for_global_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(s, i / 2, name); } - return Global(s, LINENO(n), n->n_col_offset, c->c_arena); + return Global(s, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3430,7 +3468,7 @@ ast_for_nonlocal_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(s, i / 2, name); } - return Nonlocal(s, LINENO(n), n->n_col_offset, c->c_arena); + return Nonlocal(s, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3442,7 +3480,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) expr_ty expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; - return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena); + return Assert(expression, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (NCH(n) == 4) { expr_ty expr1, expr2; @@ -3454,7 +3492,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) if (!expr2) return NULL; - return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena); + return Assert(expr1, expr2, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, "improper number of parts to 'assert' statement: %d", @@ -3550,6 +3588,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) return NULL; return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } @@ -3573,6 +3612,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) return NULL; return If(expression, seq1, seq2, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (s[2] == 'i') { @@ -3610,6 +3650,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) If(expression, suite_seq, suite_seq2, LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena)); /* the just-created orelse handled the last elif */ n_elif--; @@ -3630,7 +3671,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) asdl_seq_SET(newobj, 0, If(expression, suite_seq, orelse, LINENO(CHILD(n, off)), - CHILD(n, off)->n_col_offset, c->c_arena)); + CHILD(n, off)->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena)); orelse = newobj; } expression = ast_for_expr(c, CHILD(n, 1)); @@ -3640,7 +3681,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) if (!suite_seq) return NULL; return If(expression, suite_seq, orelse, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3664,7 +3705,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; - return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena); + return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (NCH(n) == 7) { expr_ty expression; @@ -3680,7 +3721,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) if (!seq2) return NULL; - return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena); + return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3716,7 +3757,7 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) if (NCH(node_target) == 1) target = first; else - target = Tuple(_target, Store, first->lineno, first->col_offset, c->c_arena); + target = Tuple(_target, Store, first->lineno, first->col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); expression = ast_for_testlist(c, CHILD(n, 3)); if (!expression) @@ -3728,10 +3769,12 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) if (is_async) return AsyncFor(target, expression, suite_seq, seq, LINENO(n0), n0->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } @@ -3748,7 +3791,7 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) return NULL; return ExceptHandler(NULL, NULL, suite_seq, LINENO(exc), - exc->n_col_offset, c->c_arena); + exc->n_col_offset, exc->n_end_lineno, exc->n_end_col_offset, c->c_arena); } else if (NCH(exc) == 2) { expr_ty expression; @@ -3762,7 +3805,7 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) return NULL; return ExceptHandler(expression, NULL, suite_seq, LINENO(exc), - exc->n_col_offset, c->c_arena); + exc->n_col_offset, exc->n_end_lineno, exc->n_end_col_offset, c->c_arena); } else if (NCH(exc) == 4) { asdl_seq *suite_seq; @@ -3780,7 +3823,7 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) return NULL; return ExceptHandler(expression, e, suite_seq, LINENO(exc), - exc->n_col_offset, c->c_arena); + exc->n_col_offset, exc->n_end_lineno, exc->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3850,7 +3893,7 @@ ast_for_try_stmt(struct compiling *c, const node *n) } assert(finally != NULL || asdl_seq_LEN(handlers)); - return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, c->c_arena); + return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* with_item: test ['as' expr] */ @@ -3903,9 +3946,9 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) return NULL; if (is_async) - return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, c->c_arena); + return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else - return With(items, body, LINENO(n), n->n_col_offset, c->c_arena); + return With(items, body, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3928,7 +3971,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) if (forbidden_name(c, classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ @@ -3941,7 +3984,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) if (forbidden_name(c, classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* class NAME '(' arglist ')' ':' suite */ @@ -3952,7 +3995,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) dummy_name = NEW_IDENTIFIER(CHILD(n, 1)); if (!dummy_name) return NULL; - dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena); + dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, c->c_arena); call = ast_for_call(c, CHILD(n, 3), dummy, NULL); if (!call) return NULL; @@ -3967,7 +4010,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) return NULL; return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s, - decorator_seq, LINENO(n), n->n_col_offset, c->c_arena); + decorator_seq, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3992,7 +4035,7 @@ ast_for_stmt(struct compiling *c, const node *n) case del_stmt: return ast_for_del_stmt(c, n); case pass_stmt: - return Pass(LINENO(n), n->n_col_offset, c->c_arena); + return Pass(LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); case flow_stmt: return ast_for_flow_stmt(c, n); case import_stmt: @@ -4268,6 +4311,7 @@ static void fstring_shift_node_locations(node *n, int lineno, int col_offset) static void fstring_fix_node_location(const node *parent, node *n, char *expr_str) { + // TODO: fix end positions! char *substr = NULL; char *start; int lines = LINENO(parent) - 1; @@ -4648,6 +4692,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl, entire expression with the conversion and format spec. */ *expression = FormattedValue(simple_expression, conversion, format_spec, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!*expression) return -1; @@ -4894,7 +4939,7 @@ make_str_node_and_del(PyObject **str, struct compiling *c, const node* n) Py_DECREF(s); return NULL; } - return Constant(s, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(s, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* Add a non-f-string (that is, a regular literal string). str is @@ -5049,7 +5094,7 @@ FstringParser_Finish(FstringParser *state, struct compiling *c, if (!seq) goto error; - return JoinedStr(seq, LINENO(n), n->n_col_offset, c->c_arena); + return JoinedStr(seq, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); error: FstringParser_Dealloc(state); @@ -5259,7 +5304,7 @@ parsestrplus(struct compiling *c, const node *n) /* Just return the bytes object and we're done. */ if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0) goto error; - return Constant(bytes_str, LINENO(n), n->n_col_offset, c->c_arena); + return Constant(bytes_str, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* We're not a bytes string, bytes_str should never have been set. */ diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 6f72a7f63bf922..6c0298d2b55d50 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -439,7 +439,7 @@ astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_) return 0; } asdl_seq_SET(values, 0, st->v.Expr.value); - expr_ty expr = JoinedStr(values, st->lineno, st->col_offset, ctx_); + expr_ty expr = JoinedStr(values, st->lineno, st->col_offset, st->end_lineno, st->end_col_offset, ctx_); if (!expr) { return 0; } diff --git a/Python/compile.c b/Python/compile.c index 45e78cb22cd824..e22e0e41c60664 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4710,7 +4710,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) switch (e->kind) { case Attribute_kind: auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr, - AugLoad, e->lineno, e->col_offset, c->c_arena); + AugLoad, e->lineno, e->col_offset, e->end_lineno, e->end_col_offset, c->c_arena); if (auge == NULL) return 0; VISIT(c, expr, auge); @@ -4721,7 +4721,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) break; case Subscript_kind: auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice, - AugLoad, e->lineno, e->col_offset, c->c_arena); + AugLoad, e->lineno, e->col_offset, e->end_lineno, e->end_col_offset, c->c_arena); if (auge == NULL) return 0; VISIT(c, expr, auge); From 514d4eac93f321f946b3b70d0867ed0317a7fd65 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 7 Jan 2019 00:14:39 +0000 Subject: [PATCH 04/36] Some test fixes --- Lib/ast.py | 18 ++++++++++++++---- Python/ast.c | 3 +-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 4c8c7795ff822c..f177112954b524 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -118,7 +118,7 @@ def copy_location(new_node, old_node): Copy source location (`lineno` and `col_offset` attributes) from *old_node* to *new_node* if possible, and return *new_node*. """ - for attr in 'lineno', 'col_offset': + for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': if attr in old_node._attributes and attr in new_node._attributes \ and hasattr(old_node, attr): setattr(new_node, attr, getattr(old_node, attr)) @@ -133,20 +133,30 @@ def fix_missing_locations(node): recursively where not already set, by setting them to the values of the parent node. It works recursively starting at *node*. """ - def _fix(node, lineno, col_offset): + def _fix(node, lineno, col_offset, end_lineno, end_col_offset): if 'lineno' in node._attributes: if not hasattr(node, 'lineno'): node.lineno = lineno else: lineno = node.lineno + if 'end_lineno' in node._attributes: + if not hasattr(node, 'end_lineno'): + node.end_lineno = end_lineno + else: + end_lineno = node.end_lineno if 'col_offset' in node._attributes: if not hasattr(node, 'col_offset'): node.col_offset = col_offset else: col_offset = node.col_offset + if 'end_col_offset' in node._attributes: + if not hasattr(node, 'end_col_offset'): + node.end_col_offset = end_col_offset + else: + end_col_offset = node.end_col_offset for child in iter_child_nodes(node): - _fix(child, lineno, col_offset) - _fix(node, 1, 0) + _fix(child, lineno, col_offset, end_lineno, end_col_offset) + _fix(node, 1, 0, 1, 0) return node diff --git a/Python/ast.c b/Python/ast.c index 401339cb9ef616..0aeb278586eab3 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2659,8 +2659,7 @@ ast_for_expr(struct compiling *c, const node *n) } return Compare(expression, ops, cmps, LINENO(n), - n->n_end_lineno, n->n_end_col_offset, - n->n_col_offset, c->c_arena); + n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } break; From 3ab2516e01c4f654bbd65086a85ecbe28f8ab46c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 7 Jan 2019 00:55:27 +0000 Subject: [PATCH 05/36] More test fixes --- Lib/ast.py | 2 ++ Lib/test/test_asdl_parser.py | 4 ++-- Lib/test/test_ast.py | 21 ++++++++++++--------- Lib/test/test_parser.py | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index f177112954b524..144f310bd212b4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -168,6 +168,8 @@ def increment_lineno(node, n=1): for child in walk(node): if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n + if 'end_lineno' in child._attributes: + child.end_lineno = getattr(child, 'end_lineno', 0) + n return node diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py index 15bc684904c9af..e342ece0731412 100644 --- a/Lib/test/test_asdl_parser.py +++ b/Lib/test/test_asdl_parser.py @@ -62,14 +62,14 @@ def test_product(self): def test_attributes(self): stmt = self.types['stmt'] - self.assertEqual(len(stmt.attributes), 2) + self.assertEqual(len(stmt.attributes), 4) self.assertEqual(str(stmt.attributes[0]), 'Field(int, lineno)') self.assertEqual(str(stmt.attributes[1]), 'Field(int, col_offset)') def test_constructor_fields(self): ehandler = self.types['excepthandler'] self.assertEqual(len(ehandler.types), 1) - self.assertEqual(len(ehandler.attributes), 2) + self.assertEqual(len(ehandler.attributes), 4) cons = ehandler.types[0] self.assertIsInstance(cons, self.asdl.Constructor) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index db9a6caf42f1d0..4e4a9e39cdf503 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -588,9 +588,10 @@ def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Num(2), src.body.right) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0), ' - 'op=Add(), right=Constant(value=2, lineno=1, col_offset=4), lineno=1, ' - 'col_offset=0))' + 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' + 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' + 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' + 'col_offset=0, end_lineno=1, end_col_offset=5))' ) def test_fix_missing_locations(self): @@ -613,17 +614,19 @@ def test_increment_lineno(self): src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src, n=3), src) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0), ' - 'op=Add(), right=Constant(value=1, lineno=4, col_offset=4), lineno=4, ' - 'col_offset=0))' + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' ) # issue10869: do not increment lineno of root twice src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0), ' - 'op=Add(), right=Constant(value=1, lineno=4, col_offset=4), lineno=4, ' - 'col_offset=0))' + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' ) def test_iter_fields(self): diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 274e26061a1991..9b58bb93c81c9d 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -880,7 +880,7 @@ def XXXROUNDUP(n): return 1 << (n - 1).bit_length() basesize = support.calcobjsize('Pii') - nodesize = struct.calcsize('hP3iP0h') + nodesize = struct.calcsize('hP3iP0h2i') def sizeofchildren(node): if node is None: return 0 From 1d3e352ad552c7aa243d40a15b0dde31db9b8e7e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Jan 2019 21:42:04 +0000 Subject: [PATCH 06/36] Add a TODO --- Parser/parsetok.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/parsetok.c b/Parser/parsetok.c index c8dbf0ab1dc167..3a425c6c3eacf6 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -268,7 +268,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, else { end_col_offset = -1; } - + // TODO: end line for multi-line strings (and "" \ ""). if ((err_ret->error = PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset, end_col_offset, From dbf9cc96e1a03608d1727a187356b855601f38ee Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 13 Jan 2019 19:53:41 +0000 Subject: [PATCH 07/36] Switch to better algorithm for finding end position --- Include/ast.h | 4 ++-- Include/node.h | 4 +++- Lib/test/test_ast.py | 4 ++++ Modules/parsermodule.c | 2 +- Parser/node.c | 24 ++++++++++++++++++++++-- Parser/parser.c | 21 ++++++++++++++------- Parser/parser.h | 3 ++- Parser/parsetok.c | 5 ++++- Python/ast.c | 27 ++------------------------- 9 files changed, 54 insertions(+), 40 deletions(-) diff --git a/Include/ast.h b/Include/ast.h index 545706525ab05f..f1d734852eca79 100644 --- a/Include/ast.h +++ b/Include/ast.h @@ -9,12 +9,12 @@ extern "C" { PyAPI_FUNC(int) PyAST_Validate(mod_ty); PyAPI_FUNC(mod_ty) PyAST_FromNode( - node *n, + const node *n, PyCompilerFlags *flags, const char *filename, /* decoded from the filesystem encoding */ PyArena *arena); PyAPI_FUNC(mod_ty) PyAST_FromNodeObject( - node *n, + const node *n, PyCompilerFlags *flags, PyObject *filename, PyArena *arena); diff --git a/Include/node.h b/Include/node.h index 2962bb1652175c..b2752b2bd6b070 100644 --- a/Include/node.h +++ b/Include/node.h @@ -20,7 +20,8 @@ typedef struct _node { PyAPI_FUNC(node *) PyNode_New(int type); PyAPI_FUNC(int) PyNode_AddChild(node *n, int type, - char *str, int lineno, int col_offset, int end_col_offset); + char *str, int lineno, int col_offset, + int end_lineno, int end_col_offset); PyAPI_FUNC(void) PyNode_Free(node *n); #ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); @@ -39,6 +40,7 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); #define REQ(n, type) assert(TYPE(n) == (type)) PyAPI_FUNC(void) PyNode_ListTree(node *); +void _finalize_end_pos(node *n); // helper also used in parsetok.c #ifdef __cplusplus } diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 4e4a9e39cdf503..9c19be32157301 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -238,6 +238,10 @@ def _assertTrueorder(self, ast_node, parent_pos): elif value is not None: self._assertTrueorder(value, parent_pos) + def _check_end_pos(self, ast_node, end_line, end_col_offset): + self.assertEqual(ast_node.end_line, end_line) + self.assertEqual(ast_node.end_col_offset, end_col_offset) + def test_AST_objects(self): x = ast.AST() self.assertEqual(x._fields, ()) diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index d6ef7b230b16ef..f4f5cf7cf101c5 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -920,7 +920,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_DECREF(elem); return NULL; } - err = PyNode_AddChild(root, type, strn, *line_num, 0, -1); + err = PyNode_AddChild(root, type, strn, *line_num, 0, -1, 0); if (err == E_NOMEM) { Py_DECREF(elem); PyObject_FREE(strn); diff --git a/Parser/node.c b/Parser/node.c index 6533543cc9fada..160a97b917391a 100644 --- a/Parser/node.c +++ b/Parser/node.c @@ -77,14 +77,34 @@ fancy_roundup(int n) fancy_roundup(n)) +void +_finalize_end_pos(node *n) +{ + int nch = NCH(n); + node *last; + if (nch == 0) { + return; + } + last = CHILD(n, nch - 1); + _finalize_end_pos(last); + n->n_end_lineno = last->n_end_lineno; + n->n_end_col_offset = last->n_end_col_offset; +} + int -PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset, int end_col_offset) +PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset, + int end_lineno, int end_col_offset) { const int nch = n1->n_nchildren; int current_capacity; int required_capacity; node *n; + // finalize end position of previous node (if any) + if (nch > 0) { + _finalize_end_pos(CHILD(n1, nch - 1)); + } + if (nch == INT_MAX || nch < 0) return E_OVERFLOW; @@ -109,7 +129,7 @@ PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset, int e n->n_str = str; n->n_lineno = lineno; n->n_col_offset = col_offset; - n->n_end_lineno = lineno; // this and below are correct only for terminals. + n->n_end_lineno = end_lineno; // this and below will be updates after all children are added. n->n_end_col_offset = end_col_offset; n->n_nchildren = 0; n->n_child = NULL; diff --git a/Parser/parser.c b/Parser/parser.c index c559d99b312d09..a9916d392aabb2 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -105,11 +105,13 @@ PyParser_Delete(parser_state *ps) /* PARSER STACK OPERATIONS */ static int -shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset, int end_col_offset) +shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset, + int end_lineno, int end_col_offset) { int err; assert(!s_empty(s)); - err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset, end_col_offset); + err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset, + end_lineno, end_col_offset); if (err) return err; s->s_top->s_state = newstate; @@ -117,13 +119,15 @@ shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset, i } static int -push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset, int end_col_offset) +push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset, + int end_lineno, int end_col_offset) { int err; node *n; n = s->s_top->s_parent; assert(!s_empty(s)); - err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset, end_col_offset); + err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset, + end_lineno, end_col_offset); if (err) return err; s->s_top->s_state = newstate; @@ -225,7 +229,8 @@ future_hack(parser_state *ps) int PyParser_AddToken(parser_state *ps, int type, char *str, - int lineno, int col_offset, int end_col_offset, + int lineno, int col_offset, + int end_lineno, int end_col_offset, int *expected_ret) { int ilabel; @@ -258,7 +263,8 @@ PyParser_AddToken(parser_state *ps, int type, char *str, dfa *d1 = PyGrammar_FindDFA( ps->p_grammar, nt); if ((err = push(&ps->p_stack, nt, d1, - arrow, lineno, col_offset, end_col_offset)) > 0) { + arrow, lineno, col_offset, + end_lineno, end_col_offset)) > 0) { D(printf(" MemError: push\n")); return err; } @@ -268,7 +274,8 @@ PyParser_AddToken(parser_state *ps, int type, char *str, /* Shift the token */ if ((err = shift(&ps->p_stack, type, str, - x, lineno, col_offset, end_col_offset)) > 0) { + x, lineno, col_offset, + end_lineno, end_col_offset)) > 0) { D(printf(" MemError: shift.\n")); return err; } diff --git a/Parser/parser.h b/Parser/parser.h index 373d9e417832d9..95cd39d209dd17 100644 --- a/Parser/parser.h +++ b/Parser/parser.h @@ -33,7 +33,8 @@ typedef struct { parser_state *PyParser_New(grammar *g, int start); void PyParser_Delete(parser_state *ps); int PyParser_AddToken(parser_state *ps, int type, char *str, - int lineno, int col_offset, int end_col_offset, + int lineno, int col_offset, + int end_lineno, int end_col_offset, int *expected_ret); void PyGrammar_AddAccelerators(grammar *g); diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 3a425c6c3eacf6..0b736be6fb9ca2 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -271,7 +271,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, // TODO: end line for multi-line strings (and "" \ ""). if ((err_ret->error = PyParser_AddToken(ps, (int)type, str, - tok->lineno, col_offset, end_col_offset, + tok->lineno, col_offset, tok->lineno, end_col_offset, &(err_ret->expected))) != E_OK) { if (err_ret->error != E_DONE) { PyObject_FREE(str); @@ -367,6 +367,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, done: PyTokenizer_Free(tok); + if (n != NULL) { + _finalize_end_pos(n); + } return n; } diff --git a/Python/ast.c b/Python/ast.c index 0aeb278586eab3..c68768bc9c7dff 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -739,32 +739,11 @@ num_stmts(const node *n) Py_UNREACHABLE(); } -void -update_node_ends(node *n) -{ - int max_line = 0, max_col = -1; - node *ch; - - if (NCH(n) == 0) { - return; - } - - for (int i = 0; i < NCH(n); i++) { - ch = CHILD(n, i); - update_node_ends(ch); - max_line = ch->n_end_lineno > max_line ? ch->n_end_lineno : max_line; - max_col = ch->n_end_col_offset > max_col ? ch->n_end_col_offset : max_col; - } - n->n_end_lineno = max_line; - n->n_end_col_offset = max_col; -} - - /* Transform the CST rooted at node * to the appropriate AST */ mod_ty -PyAST_FromNodeObject(node *n, PyCompilerFlags *flags, +PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, PyObject *filename, PyArena *arena) { int i, j, k, num; @@ -779,8 +758,6 @@ PyAST_FromNodeObject(node *n, PyCompilerFlags *flags, c.c_filename = filename; c.c_normalize = NULL; - update_node_ends(n); - if (TYPE(n) == encoding_decl) n = CHILD(n, 0); @@ -878,7 +855,7 @@ PyAST_FromNodeObject(node *n, PyCompilerFlags *flags, } mod_ty -PyAST_FromNode(node *n, PyCompilerFlags *flags, const char *filename_str, +PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename_str, PyArena *arena) { mod_ty mod; From 5af33da74d7b35f4734102ff45fb9ab23820c532 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 13 Jan 2019 20:20:38 +0000 Subject: [PATCH 08/36] Be consistent for line_num --- Modules/parsermodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index f4f5cf7cf101c5..eabc5c810f6ce6 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -920,7 +920,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_DECREF(elem); return NULL; } - err = PyNode_AddChild(root, type, strn, *line_num, 0, -1, 0); + err = PyNode_AddChild(root, type, strn, *line_num, 0, *line_num, 0); if (err == E_NOMEM) { Py_DECREF(elem); PyObject_FREE(strn); From ce7f5ce83ad050b17dcaf17697545d566b9d0b71 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 13 Jan 2019 23:33:07 +0000 Subject: [PATCH 09/36] Minor fixes; start adding tests --- Lib/test/test_ast.py | 77 +++++++++++++++++++++++++++++++++++++++++--- Python/ast.c | 64 ++++++++++++++++++++++++++++-------- 2 files changed, 123 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 9b6ecfbd66d4ec..41552ae25e99b2 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -4,6 +4,7 @@ import sys import unittest import weakref +from textwrap import dedent from test import support @@ -238,10 +239,6 @@ def _assertTrueorder(self, ast_node, parent_pos): elif value is not None: self._assertTrueorder(value, parent_pos) - def _check_end_pos(self, ast_node, end_line, end_col_offset): - self.assertEqual(ast_node.end_line, end_line) - self.assertEqual(ast_node.end_col_offset, end_col_offset) - def test_AST_objects(self): x = ast.AST() self.assertEqual(x._fields, ()) @@ -1276,6 +1273,78 @@ def test_literal_eval(self): self.assertEqual(ast.literal_eval(binop), 10+20j) +class EndPositionTests(unittest.TestCase): + """Tests for end position of AST nodes. + + Testing end positions of nodes requires a bit of extra care + because of how LL parsers work. + """ + def _check_end_pos(self, ast_node, end_lineno, end_col_offset): + self.assertEqual(ast_node.end_lineno, end_lineno) + self.assertEqual(ast_node.end_col_offset, end_col_offset) + + def _check_content(self, source, ast_node, content): + # should be used for one line expressions only. + assert ast_node.lineno == ast_node.end_lineno + self.assertEqual(source.splitlines()[ast_node.lineno - 1] # lines are 1-based + [ast_node.col_offset:ast_node.end_col_offset], + content) + + def _parse_expr(self, s): + return ast.parse(s).body[0].value + + def test_lambda(self): + s = 'lambda x, *y: None' + lam = self._parse_expr(s) + self._check_content(s, lam.body, 'None') + self._check_content(s, lam.args.args[0], 'x') + self._check_content(s, lam.args.vararg, 'y') + + def test_func_def(self): + s = dedent(''' + def func(x: int, + *args: str, + z: float = 0, + **kwargs: Any) -> bool: + return True + ''').strip() + fdef = ast.parse(s).body[0] + self._check_end_pos(fdef, 5, 15) + self._check_content(s, fdef.body[0], 'return True') + self._check_content(s, fdef.args.args[0], 'x: int') + self._check_content(s, fdef.args.args[0].annotation, 'int') + self._check_content(s, fdef.args.kwarg, 'kwargs: Any') + self._check_content(s, fdef.args.kwarg.annotation, 'Any') + + def test_call(self): + s = 'func(x, y=2, **kw)' + call = self._parse_expr(s) + self._check_content(s, call.func, 'func') + self._check_content(s, call.keywords[0].value, '2') + self._check_content(s, call.keywords[1].value, 'kw') + + def test_call_noargs(self): + s = 'x[0]()' + call = self._parse_expr(s) + self._check_content(s, call.func, 'x[0]') + self._check_end_pos(call, 1, 6) + + def test_class_def(self): + s = dedent(''' + class C(A, B): + x: int = 0 + ''').strip() + cdef = ast.parse(s).body[0] + self._check_end_pos(cdef, 2, 14) + self._check_content(s, cdef.bases[1], 'B') + self._check_content(s, cdef.body[0], 'x: int = 0') + + def test_class_kw(self): + s = 'class S(metaclass=abc.ABCMeta): pass' + cdef = ast.parse(s).body[0] + self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') + + def main(): if __name__ != '__main__': return diff --git a/Python/ast.c b/Python/ast.c index a80ea99a870a64..4e4b053d415a52 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -580,7 +580,7 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, bool); /* Note different signature for ast_for_call */ static expr_ty ast_for_call(struct compiling *, const node *, expr_ty, - const node *); + const node *, const node *); static PyObject *parsenumber(struct compiling *, const char *); static expr_ty parsestrplus(struct compiling *, const node *n); @@ -1538,7 +1538,7 @@ ast_for_decorator(struct compiling *c, const node *n) name_expr = NULL; } else { - d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2)); + d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2), CHILD(n, 4)); if (!d) return NULL; name_expr = NULL; @@ -1579,6 +1579,8 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, asdl_seq *body; expr_ty returns = NULL; int name_i = 1; + int tot, end_lineno, end_col_offset; + stmt_ty last; REQ(n, funcdef); @@ -1600,12 +1602,17 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, if (!body) return NULL; + tot = asdl_seq_LEN(body); + last = asdl_seq_GET(body, tot - 1); + end_lineno = last->end_lineno; + end_col_offset = last->end_col_offset; + if (is_async) return AsyncFunctionDef(name, args, body, decorator_seq, returns, - LINENO(n0), n0->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else return FunctionDef(name, args, body, decorator_seq, returns, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } static stmt_ty @@ -2349,13 +2356,14 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) subscriptlist: subscript (',' subscript)* [','] subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] */ + const node *n_copy = n; REQ(n, trailer); if (TYPE(CHILD(n, 0)) == LPAR) { if (NCH(n) == 2) return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else - return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0)); + return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0), CHILD(n, 2)); } else if (TYPE(CHILD(n, 0)) == DOT) { PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1)); @@ -2373,7 +2381,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!slc) return NULL; return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, + n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } else { @@ -2399,7 +2407,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) } if (!simple) { return Subscript(left_expr, ExtSlice(slices, c->c_arena), - Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + Load, LINENO(n), n->n_col_offset, + n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } /* extract Index values and put them in a Tuple */ elts = _Py_asdl_seq_new(asdl_seq_LEN(slices), c->c_arena); @@ -2414,7 +2423,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!e) return NULL; return Subscript(left_expr, Index(e, c->c_arena), - Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + Load, LINENO(n), n->n_col_offset, + n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } } } @@ -2699,7 +2709,7 @@ ast_for_expr(struct compiling *c, const node *n) static expr_ty ast_for_call(struct compiling *c, const node *n, expr_ty func, - const node *maybegenbeg) + const node *maybegenbeg, const node *closepar) { /* arglist: argument (',' argument)* [','] @@ -2708,6 +2718,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, int i, nargs, nkeywords; int ndoublestars; + int end_lineno, end_col_offset; asdl_seq *args; asdl_seq *keywords; @@ -2880,7 +2891,11 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, } } - return Call(func, args, keywords, func->lineno, func->col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + // Ideally call ends where closing parenthesis is placed. + end_lineno = closepar == NULL ? n->n_end_lineno : closepar->n_end_lineno; + end_col_offset = closepar == NULL ? n->n_end_col_offset : closepar->n_end_col_offset; + + return Call(func, args, keywords, func->lineno, func->col_offset, end_lineno, end_col_offset, c->c_arena); } static expr_ty @@ -3936,6 +3951,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) PyObject *classname; asdl_seq *s; expr_ty call; + int end_lineno, end_col_offset, tot; + stmt_ty last; REQ(n, classdef); @@ -3943,26 +3960,39 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) s = ast_for_suite(c, CHILD(n, 3)); if (!s) return NULL; + + // TODO: refactor this whole function to avoid code duplication. + tot = asdl_seq_LEN(s); + last = asdl_seq_GET(s, tot - 1); + end_lineno = last->end_lineno; + end_col_offset = last->end_col_offset; + classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) return NULL; if (forbidden_name(c, classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ s = ast_for_suite(c, CHILD(n, 5)); if (!s) return NULL; + + tot = asdl_seq_LEN(s); + last = asdl_seq_GET(s, tot - 1); + end_lineno = last->end_lineno; + end_col_offset = last->end_col_offset; + classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) return NULL; if (forbidden_name(c, classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } /* class NAME '(' arglist ')' ':' suite */ @@ -3974,13 +4004,19 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) if (!dummy_name) return NULL; dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, c->c_arena); - call = ast_for_call(c, CHILD(n, 3), dummy, NULL); + call = ast_for_call(c, CHILD(n, 3), dummy, NULL, NULL); if (!call) return NULL; } s = ast_for_suite(c, CHILD(n, 6)); if (!s) return NULL; + + tot = asdl_seq_LEN(s); + last = asdl_seq_GET(s, tot - 1); + end_lineno = last->end_lineno; + end_col_offset = last->end_col_offset; + classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) return NULL; @@ -3988,7 +4024,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) return NULL; return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s, - decorator_seq, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + decorator_seq, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } static stmt_ty From 2171eb9dd1b842995bddd91aece1183e13caaed6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 13 Jan 2019 23:56:00 +0000 Subject: [PATCH 10/36] Update two failing tests --- Lib/test/test_ast.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 41552ae25e99b2..84b9a9c2e5e992 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -579,10 +579,12 @@ def test_dump(self): ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), " - "lineno=1, col_offset=5), Constant(value='and cheese', lineno=1, " - "col_offset=11)], keywords=[], " - "lineno=1, col_offset=0), lineno=1, col_offset=0)])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " + "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " + "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " + "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) def test_copy_location(self): @@ -600,15 +602,18 @@ def test_fix_missing_locations(self): src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), [ast.Str('eggs')], []))) self.assertEqual(src, ast.fix_missing_locations(src)) + self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " - "lineno=1, col_offset=0), args=[Constant(value='spam', lineno=1, " - "col_offset=6)], keywords=[], " - "lineno=1, col_offset=0), lineno=1, col_offset=0), " - "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, " - "col_offset=0), args=[Constant(value='eggs', lineno=1, col_offset=0)], " - "keywords=[], lineno=1, " - "col_offset=0), lineno=1, col_offset=0)])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " + "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " + "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " + "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) def test_increment_lineno(self): From 10cf4bd6b247bfb695b6c7d578b6d081d8f2381f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 14 Jan 2019 23:58:20 +0000 Subject: [PATCH 11/36] Fix multiline strings --- Lib/test/test_ast.py | 19 +++++++++++++++++++ Parser/parsetok.c | 5 ++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 84b9a9c2e5e992..b54b0630df6b70 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1349,6 +1349,25 @@ def test_class_kw(self): cdef = ast.parse(s).body[0] self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') + def test_multi_line_str(self): + s = dedent(''' + x = """Some multi-line text. + + It goes on starting from same indent.""" + ''').strip() + assign = ast.parse(s).body[0] + self._check_end_pos(assign, 3, 40) + self._check_end_pos(assign.value, 3, 40) + + def test_continued_str(self): + s = dedent(''' + x = "first part" \\ + "second part" + ''').strip() + assign = ast.parse(s).body[0] + self._check_end_pos(assign, 2, 13) + self._check_end_pos(assign.value, 2, 13) + def main(): if __name__ != '__main__': diff --git a/Parser/parsetok.c b/Parser/parsetok.c index c3f7ddf8ec11c1..ea0cf7c9155c2a 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -271,16 +271,15 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, } if (b != NULL && b >= line_start) { - end_col_offset = Py_SAFE_DOWNCAST(b - line_start, + end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start, intptr_t, int); } else { end_col_offset = -1; } - // TODO: end line for multi-line strings (and "" \ ""). if ((err_ret->error = PyParser_AddToken(ps, (int)type, str, - lineno, col_offset, lineno, end_col_offset, + lineno, col_offset, tok->lineno, end_col_offset, &(err_ret->expected))) != E_OK) { if (err_ret->error != E_DONE) { PyObject_FREE(str); From ed053052184bb164b206f03e4a84a96c324c4f91 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Jan 2019 01:00:34 +0000 Subject: [PATCH 12/36] Fix end position for if statement --- Lib/test/test_ast.py | 25 +++++++++++++++++ Python/ast.c | 66 ++++++++++++++++++++++---------------------- 2 files changed, 58 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index b54b0630df6b70..16b6ad07df60b8 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1368,6 +1368,31 @@ def test_continued_str(self): self._check_end_pos(assign, 2, 13) self._check_end_pos(assign.value, 2, 13) + def test_control_stmts(self): + # We intentionally put these into the same string to check + # that empty lines are not part of the suite. + s = dedent(''' + while True: + pass + + if one(): + x = None + elif other(): + y = None + else: + z = None + + for x, y in stuff: + assert True + + try: + raise RuntimeError + except TypeError as e: + pass + + pass + ''').strip() + def main(): if __name__ != '__main__': diff --git a/Python/ast.c b/Python/ast.c index 4e4b053d415a52..64c109dee23830 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -584,6 +584,7 @@ static expr_ty ast_for_call(struct compiling *, const node *, expr_ty, static PyObject *parsenumber(struct compiling *, const char *); static expr_ty parsestrplus(struct compiling *, const node *n); +static void get_last_end_pos(asdl_seq *, int *, int *); #define COMP_GENEXP 0 #define COMP_LISTCOMP 1 @@ -1579,8 +1580,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, asdl_seq *body; expr_ty returns = NULL; int name_i = 1; - int tot, end_lineno, end_col_offset; - stmt_ty last; + int end_lineno, end_col_offset; REQ(n, funcdef); @@ -1601,11 +1601,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, body = ast_for_suite(c, CHILD(n, name_i + 3)); if (!body) return NULL; - - tot = asdl_seq_LEN(body); - last = asdl_seq_GET(body, tot - 1); - end_lineno = last->end_lineno; - end_col_offset = last->end_col_offset; + get_last_end_pos(body, &end_lineno, &end_col_offset); if (is_async) return AsyncFunctionDef(name, args, body, decorator_seq, returns, @@ -3559,6 +3555,15 @@ ast_for_suite(struct compiling *c, const node *n) return seq; } +static void +get_last_end_pos(asdl_seq *s, int *end_lineno, int *end_col_offset) +{ + int tot = asdl_seq_LEN(s); + stmt_ty last = asdl_seq_GET(s, tot - 1); + *end_lineno = last->end_lineno; + *end_col_offset = last->end_col_offset; +} + static stmt_ty ast_for_if_stmt(struct compiling *c, const node *n) { @@ -3566,6 +3571,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) ['else' ':' suite] */ char *s; + int end_lineno, end_col_offset; REQ(n, if_stmt); @@ -3579,10 +3585,10 @@ ast_for_if_stmt(struct compiling *c, const node *n) suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, - c->c_arena); + end_lineno, end_col_offset, c->c_arena); } s = STR(CHILD(n, 4)); @@ -3603,10 +3609,10 @@ ast_for_if_stmt(struct compiling *c, const node *n) seq2 = ast_for_suite(c, CHILD(n, 6)); if (!seq2) return NULL; + get_last_end_pos(seq2, &end_lineno, &end_col_offset); return If(expression, seq1, seq2, LINENO(n), n->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, - c->c_arena); + end_lineno, end_col_offset, c->c_arena); } else if (s[2] == 'i') { int i, n_elif, has_else = 0; @@ -3638,13 +3644,13 @@ ast_for_if_stmt(struct compiling *c, const node *n) suite_seq2 = ast_for_suite(c, CHILD(n, NCH(n) - 1)); if (!suite_seq2) return NULL; + get_last_end_pos(suite_seq2, &end_lineno, &end_col_offset); asdl_seq_SET(orelse, 0, If(expression, suite_seq, suite_seq2, LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, - c->c_arena)); + end_lineno, end_col_offset, c->c_arena)); /* the just-created orelse handled the last elif */ n_elif--; } @@ -3661,10 +3667,16 @@ ast_for_if_stmt(struct compiling *c, const node *n) if (!suite_seq) return NULL; + if (orelse != NULL) { + get_last_end_pos(orelse, &end_lineno, &end_col_offset); + } else { + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); + } asdl_seq_SET(newobj, 0, If(expression, suite_seq, orelse, LINENO(CHILD(n, off)), - CHILD(n, off)->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena)); + CHILD(n, off)->n_col_offset, + end_lineno, end_col_offset, c->c_arena)); orelse = newobj; } expression = ast_for_expr(c, CHILD(n, 1)); @@ -3673,8 +3685,10 @@ ast_for_if_stmt(struct compiling *c, const node *n) suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; + get_last_end_pos(orelse, &end_lineno, &end_col_offset); return If(expression, suite_seq, orelse, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3951,8 +3965,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) PyObject *classname; asdl_seq *s; expr_ty call; - int end_lineno, end_col_offset, tot; - stmt_ty last; + int end_lineno, end_col_offset; REQ(n, classdef); @@ -3960,12 +3973,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) s = ast_for_suite(c, CHILD(n, 3)); if (!s) return NULL; - - // TODO: refactor this whole function to avoid code duplication. - tot = asdl_seq_LEN(s); - last = asdl_seq_GET(s, tot - 1); - end_lineno = last->end_lineno; - end_col_offset = last->end_col_offset; + get_last_end_pos(s, &end_lineno, &end_col_offset); classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) @@ -3980,11 +3988,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) s = ast_for_suite(c, CHILD(n, 5)); if (!s) return NULL; - - tot = asdl_seq_LEN(s); - last = asdl_seq_GET(s, tot - 1); - end_lineno = last->end_lineno; - end_col_offset = last->end_col_offset; + get_last_end_pos(s, &end_lineno, &end_col_offset); classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) @@ -4011,11 +4015,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) s = ast_for_suite(c, CHILD(n, 6)); if (!s) return NULL; - - tot = asdl_seq_LEN(s); - last = asdl_seq_GET(s, tot - 1); - end_lineno = last->end_lineno; - end_col_offset = last->end_col_offset; + get_last_end_pos(s, &end_lineno, &end_col_offset); classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) From 58fbfa67c19e3dd9eb8402e0d610a756b67a4217 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Jan 2019 01:16:42 +0000 Subject: [PATCH 13/36] Adjust end positions in while and for --- Python/ast.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 64c109dee23830..ada8631a0978e4 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3701,6 +3701,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) { /* while_stmt: 'while' test ':' suite ['else' ':' suite] */ REQ(n, while_stmt); + int end_lineno, end_col_offset; if (NCH(n) == 4) { expr_ty expression; @@ -3712,7 +3713,9 @@ ast_for_while_stmt(struct compiling *c, const node *n) suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; - return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); + return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } else if (NCH(n) == 7) { expr_ty expression; @@ -3727,8 +3730,10 @@ ast_for_while_stmt(struct compiling *c, const node *n) seq2 = ast_for_suite(c, CHILD(n, 6)); if (!seq2) return NULL; + get_last_end_pos(seq2, &end_lineno, &end_col_offset); - return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3745,6 +3750,7 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) expr_ty expression; expr_ty target, first; const node *node_target; + int end_lineno, end_col_offset; /* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */ REQ(n, for_stmt); @@ -3764,7 +3770,9 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) if (NCH(node_target) == 1) target = first; else - target = Tuple(_target, Store, first->lineno, first->col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + target = Tuple(_target, Store, first->lineno, first->col_offset, + node_target->n_end_lineno, node_target->n_end_col_offset, + c->c_arena); expression = ast_for_testlist(c, CHILD(n, 3)); if (!expression) @@ -3773,16 +3781,19 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) if (!suite_seq) return NULL; + if (seq != NULL) { + get_last_end_pos(seq, &end_lineno, &end_col_offset); + } else { + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); + } if (is_async) return AsyncFor(target, expression, suite_seq, seq, LINENO(n0), n0->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, - c->c_arena); + end_lineno, end_col_offset, c->c_arena); else return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, - c->c_arena); + end_lineno, end_col_offset, c->c_arena); } static excepthandler_ty From f2589ff131b0296c61dd6ed1e27078bb1603503c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Jan 2019 01:20:05 +0000 Subject: [PATCH 14/36] Add also with --- Python/ast.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index ada8631a0978e4..bc8fc8c869c299 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3943,7 +3943,7 @@ static stmt_ty ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) { const node * const n = is_async ? CHILD(n0, 1) : n0; - int i, n_items; + int i, n_items, end_lineno, end_col_offset; asdl_seq *items, *body; REQ(n, with_stmt); @@ -3962,11 +3962,14 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) body = ast_for_suite(c, CHILD(n, NCH(n) - 1)); if (!body) return NULL; + get_last_end_pos(body, &end_lineno, &end_col_offset); if (is_async) - return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, + end_lineno, end_col_offset, c->c_arena); else - return With(items, body, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return With(items, body, LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } static stmt_ty From 7d5ca5ea7477aa56c320bea8c50d2feab2cfc36f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Jan 2019 02:05:51 +0000 Subject: [PATCH 15/36] Fix try end position (concludes fixing suites) --- Python/ast.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index bc8fc8c869c299..9de3dc83222406 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3800,6 +3800,7 @@ static excepthandler_ty ast_for_except_clause(struct compiling *c, const node *exc, node *body) { /* except_clause: 'except' [test ['as' test]] */ + int end_lineno, end_col_offset; REQ(exc, except_clause); REQ(body, suite); @@ -3807,9 +3808,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) asdl_seq *suite_seq = ast_for_suite(c, body); if (!suite_seq) return NULL; + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); return ExceptHandler(NULL, NULL, suite_seq, LINENO(exc), - exc->n_col_offset, exc->n_end_lineno, exc->n_end_col_offset, c->c_arena); + exc->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } else if (NCH(exc) == 2) { expr_ty expression; @@ -3821,9 +3824,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) suite_seq = ast_for_suite(c, body); if (!suite_seq) return NULL; + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); return ExceptHandler(expression, NULL, suite_seq, LINENO(exc), - exc->n_col_offset, exc->n_end_lineno, exc->n_end_col_offset, c->c_arena); + exc->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } else if (NCH(exc) == 4) { asdl_seq *suite_seq; @@ -3839,9 +3844,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) suite_seq = ast_for_suite(c, body); if (!suite_seq) return NULL; + get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); return ExceptHandler(expression, e, suite_seq, LINENO(exc), - exc->n_col_offset, exc->n_end_lineno, exc->n_end_col_offset, c->c_arena); + exc->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3854,8 +3861,9 @@ static stmt_ty ast_for_try_stmt(struct compiling *c, const node *n) { const int nch = NCH(n); - int n_except = (nch - 3)/3; + int end_lineno, end_col_offset, n_except = (nch - 3)/3; asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL; + excepthandler_ty last_handler; REQ(n, try_stmt); @@ -3911,7 +3919,20 @@ ast_for_try_stmt(struct compiling *c, const node *n) } assert(finally != NULL || asdl_seq_LEN(handlers)); - return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + if (finally != NULL) { + // finally is always last + get_last_end_pos(finally, &end_lineno, &end_col_offset); + } else if (orelse != NULL) { + // otherwise else is last + get_last_end_pos(orelse, &end_lineno, &end_col_offset); + } else { + // inline the get_last_end_pos logic due to layout mismatch + last_handler = (excepthandler_ty) asdl_seq_GET(handlers, n_except - 1); + end_lineno = last_handler->end_lineno; + end_col_offset = last_handler->end_col_offset; + } + return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } /* with_item: test ['as' expr] */ From aa62e3c70a3e55eac4663575628e591530b937a3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Jan 2019 10:34:03 +0000 Subject: [PATCH 16/36] Some formatting plus minor fixes --- Python/ast.c | 75 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 9de3dc83222406..9dfd84d4d626bc 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1232,7 +1232,8 @@ ast_for_arg(struct compiling *c, const node *n) return NULL; } - ret = arg(name, annotation, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + ret = arg(name, annotation, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!ret) return NULL; return ret; @@ -1496,7 +1497,8 @@ ast_for_dotted_name(struct compiling *c, const node *n) id = NEW_IDENTIFIER(ch); if (!id) return NULL; - e = Name(id, Load, lineno, col_offset, ch->n_end_lineno, ch->n_end_col_offset, c->c_arena); + e = Name(id, Load, lineno, col_offset, + ch->n_end_lineno, ch->n_end_col_offset, c->c_arena); if (!e) return NULL; @@ -1504,7 +1506,8 @@ ast_for_dotted_name(struct compiling *c, const node *n) id = NEW_IDENTIFIER(CHILD(n, i)); if (!id) return NULL; - e = Attribute(e, id, Load, lineno, col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + e = Attribute(e, id, Load, lineno, col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!e) return NULL; } @@ -1713,7 +1716,8 @@ ast_for_lambdef(struct compiling *c, const node *n) return NULL; } - return Lambda(args, expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Lambda(args, expression, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -1862,8 +1866,9 @@ ast_for_comprehension(struct compiling *c, const node *n) comp = comprehension(first, expression, NULL, is_async, c->c_arena); else - comp = comprehension(Tuple(t, Store, first->lineno, - first->col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena), + comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset, + for_ch->n_end_lineno, for_ch->n_end_col_offset, + c->c_arena), expression, NULL, is_async, c->c_arena); if (!comp) return NULL; @@ -1928,11 +1933,14 @@ ast_for_itercomp(struct compiling *c, const node *n, int type) return NULL; if (type == COMP_GENEXP) - return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else if (type == COMP_LISTCOMP) - return ListComp(elt, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return ListComp(elt, comps, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else if (type == COMP_SETCOMP) - return SetComp(elt, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return SetComp(elt, comps, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else /* Should never happen */ return NULL; @@ -1994,7 +2002,8 @@ ast_for_dictcomp(struct compiling *c, const node *n) if (!comps) return NULL; - return DictComp(key, value, comps, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return DictComp(key, value, comps, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2027,7 +2036,8 @@ ast_for_dictdisplay(struct compiling *c, const node *n) } keys->size = j; values->size = j; - return Dict(keys, values, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Dict(keys, values, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2070,7 +2080,8 @@ ast_for_setdisplay(struct compiling *c, const node *n) return NULL; asdl_seq_SET(elts, i / 2, expression); } - return Set(elts, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Set(elts, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2089,17 +2100,21 @@ ast_for_atom(struct compiling *c, const node *n) size_t len = strlen(s); if (len >= 4 && len <= 5) { if (!strcmp(s, "None")) - return Constant(Py_None, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(Py_None, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!strcmp(s, "True")) - return Constant(Py_True, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(Py_True, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!strcmp(s, "False")) - return Constant(Py_False, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(Py_False, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } name = new_identifier(s, c); if (!name) return NULL; /* All names start in Load context, but may later be changed. */ - return Name(name, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Name(name, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case STRING: { expr_ty str = parsestrplus(c, n); @@ -2138,15 +2153,18 @@ ast_for_atom(struct compiling *c, const node *n) Py_DECREF(pynum); return NULL; } - return Constant(pynum, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(pynum, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case ELLIPSIS: /* Ellipsis */ - return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); if (TYPE(ch) == RPAR) - return Tuple(NULL, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Tuple(NULL, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); @@ -2166,7 +2184,8 @@ ast_for_atom(struct compiling *c, const node *n) ch = CHILD(n, 1); if (TYPE(ch) == RSQB) - return List(NULL, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return List(NULL, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); REQ(ch, testlist_comp); if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { @@ -2174,7 +2193,8 @@ ast_for_atom(struct compiling *c, const node *n) if (!elts) return NULL; - return List(elts, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return List(elts, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { return copy_location(ast_for_listcomp(c, ch), n); @@ -2188,7 +2208,8 @@ ast_for_atom(struct compiling *c, const node *n) ch = CHILD(n, 1); if (TYPE(ch) == RBRACE) { /* It's an empty dict. */ - return Dict(NULL, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Dict(NULL, NULL, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { int is_dict = (TYPE(CHILD(ch, 0)) == DOUBLESTAR); @@ -2336,7 +2357,8 @@ ast_for_binop(struct compiling *c, const node *n) tmp_result = BinOp(result, newoperator, tmp, LINENO(next_oper), next_oper->n_col_offset, - CHILD(n, i * 2 + 2)->n_end_lineno, CHILD(n, i * 2 + 2)->n_end_col_offset, + CHILD(n, i * 2 + 2)->n_end_lineno, + CHILD(n, i * 2 + 2)->n_end_col_offset, c->c_arena); if (!tmp_result) return NULL; @@ -2356,8 +2378,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) REQ(n, trailer); if (TYPE(CHILD(n, 0)) == LPAR) { if (NCH(n) == 2) - return Call(left_expr, NULL, NULL, LINENO(n), - n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0), CHILD(n, 2)); } @@ -2366,7 +2388,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!attr_id) return NULL; return Attribute(left_expr, attr_id, Load, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { REQ(CHILD(n, 0), LSQB); From 96a0ec098529f20b702badac6ef9a2644154b02d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 16 Jan 2019 01:18:17 +0000 Subject: [PATCH 17/36] More formatting; fix import from --- Python/ast.c | 112 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 38 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 9dfd84d4d626bc..f4206285f1a1e2 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2438,7 +2438,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) assert(slc->kind == Index_kind && slc->v.Index.value); asdl_seq_SET(elts, j, slc->v.Index.value); } - e = Tuple(elts, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + e = Tuple(elts, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!e) return NULL; return Subscript(left_expr, Index(e, c->c_arena), @@ -2467,9 +2468,9 @@ ast_for_factor(struct compiling *c, const node *n) n->n_end_lineno, n->n_end_col_offset, c->c_arena); case TILDE: - return UnaryOp(Invert, expression, LINENO(n), + return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, - n->n_col_offset, c->c_arena); + c->c_arena); } PyErr_Format(PyExc_SystemError, "unhandled factor: %d", TYPE(CHILD(n, 0))); @@ -2496,7 +2497,8 @@ ast_for_atom_expr(struct compiling *c, const node *n) if (nch == 1) return e; if (start && nch == 2) { - return Await(e, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Await(e, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } for (i = start + 1; i < nch; i++) { @@ -2513,7 +2515,8 @@ ast_for_atom_expr(struct compiling *c, const node *n) if (start) { /* there was an 'await' */ - return Await(e, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Await(e, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { return e; @@ -2536,7 +2539,8 @@ ast_for_power(struct compiling *c, const node *n) expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1)); if (!f) return NULL; - e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } return e; } @@ -2552,7 +2556,8 @@ ast_for_starred(struct compiling *c, const node *n) return NULL; /* The Load context is changed later. */ - return Starred(tmp, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Starred(tmp, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } @@ -2614,7 +2619,8 @@ ast_for_expr(struct compiling *c, const node *n) n->n_end_lineno, n->n_end_col_offset, c->c_arena); assert(!strcmp(STR(CHILD(n, 1)), "or")); - return BoolOp(Or, seq, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return BoolOp(Or, seq, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case not_test: if (NCH(n) == 1) { n = CHILD(n, 0); @@ -2666,8 +2672,8 @@ ast_for_expr(struct compiling *c, const node *n) return NULL; } - return Compare(expression, ops, cmps, LINENO(n), - n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } break; @@ -2707,8 +2713,10 @@ ast_for_expr(struct compiling *c, const node *n) return NULL; } if (is_from) - return YieldFrom(exp, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); - return Yield(exp, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return YieldFrom(exp, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Yield(exp, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case factor: if (NCH(n) == 1) { @@ -2914,7 +2922,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, end_lineno = closepar == NULL ? n->n_end_lineno : closepar->n_end_lineno; end_col_offset = closepar == NULL ? n->n_end_col_offset : closepar->n_end_col_offset; - return Call(func, args, keywords, func->lineno, func->col_offset, end_lineno, end_col_offset, c->c_arena); + return Call(func, args, keywords, func->lineno, func->col_offset, + end_lineno, end_col_offset, c->c_arena); } static expr_ty @@ -2937,7 +2946,8 @@ ast_for_testlist(struct compiling *c, const node* n) asdl_seq *tmp = seq_for_testlist(c, n); if (!tmp) return NULL; - return Tuple(tmp, Load, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Tuple(tmp, Load, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } @@ -2959,7 +2969,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!e) return NULL; - return Expr(e, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Expr(e, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (TYPE(CHILD(n, 1)) == augassign) { expr_ty expr1, expr2; @@ -2997,7 +3008,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!newoperator) return NULL; - return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (TYPE(CHILD(n, 1)) == annassign) { expr_ty expr1, expr2, expr3; @@ -3057,7 +3069,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n) } if (NCH(ann) == 2) { return AnnAssign(expr1, expr2, NULL, simple, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { ch = CHILD(ann, 3); @@ -3066,7 +3079,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n) return NULL; } return AnnAssign(expr1, expr2, expr3, simple, - LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } else { @@ -3104,7 +3118,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n) expression = ast_for_expr(c, value); if (!expression) return NULL; - return Assign(targets, expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Assign(targets, expression, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } @@ -3143,7 +3158,8 @@ ast_for_del_stmt(struct compiling *c, const node *n) expr_list = ast_for_exprlist(c, CHILD(n, 1), Del); if (!expr_list) return NULL; - return Delete(expr_list, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Delete(expr_list, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3165,27 +3181,33 @@ ast_for_flow_stmt(struct compiling *c, const node *n) ch = CHILD(n, 0); switch (TYPE(ch)) { case break_stmt: - return Break(LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Break(LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case continue_stmt: - return Continue(LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Continue(LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case yield_stmt: { /* will reduce to yield_expr */ expr_ty exp = ast_for_expr(c, CHILD(ch, 0)); if (!exp) return NULL; - return Expr(exp, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Expr(exp, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case return_stmt: if (NCH(ch) == 1) - return Return(NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Return(NULL, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else { expr_ty expression = ast_for_testlist(c, CHILD(ch, 1)); if (!expression) return NULL; - return Return(expression, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Return(expression, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case raise_stmt: if (NCH(ch) == 1) - return Raise(NULL, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Raise(NULL, NULL, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); else if (NCH(ch) >= 2) { expr_ty cause = NULL; expr_ty expression = ast_for_expr(c, CHILD(ch, 1)); @@ -3196,7 +3218,8 @@ ast_for_flow_stmt(struct compiling *c, const node *n) if (!cause) return NULL; } - return Raise(expression, cause, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Raise(expression, cause, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* fall through */ default: @@ -3357,11 +3380,14 @@ ast_for_import_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(aliases, i / 2, import_alias); } - return Import(aliases, lineno, col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + // Even though n is modified above, the end position is not changed + return Import(aliases, lineno, col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (TYPE(n) == import_from) { int n_children; int idx, ndots = 0; + const node *n_copy = n; alias_ty mod = NULL; identifier modname = NULL; @@ -3432,7 +3458,7 @@ ast_for_import_stmt(struct compiling *c, const node *n) if (mod != NULL) modname = mod->name; return ImportFrom(modname, aliases, ndots, lineno, col_offset, - n->n_end_lineno, n->n_end_col_offset, + n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -3459,7 +3485,8 @@ ast_for_global_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(s, i / 2, name); } - return Global(s, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Global(s, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3480,7 +3507,8 @@ ast_for_nonlocal_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(s, i / 2, name); } - return Nonlocal(s, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Nonlocal(s, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static stmt_ty @@ -3492,7 +3520,8 @@ ast_for_assert_stmt(struct compiling *c, const node *n) expr_ty expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; - return Assert(expression, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Assert(expression, NULL, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else if (NCH(n) == 4) { expr_ty expr1, expr2; @@ -3504,7 +3533,8 @@ ast_for_assert_stmt(struct compiling *c, const node *n) if (!expr2) return NULL; - return Assert(expr1, expr2, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Assert(expr1, expr2, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, "improper number of parts to 'assert' statement: %d", @@ -4039,7 +4069,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) if (forbidden_name(c, classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ @@ -4054,7 +4085,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) if (forbidden_name(c, classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); + LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } /* class NAME '(' arglist ')' ':' suite */ @@ -4065,7 +4097,9 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) dummy_name = NEW_IDENTIFIER(CHILD(n, 1)); if (!dummy_name) return NULL; - dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, c->c_arena); + dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, + CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, + c->c_arena); call = ast_for_call(c, CHILD(n, 3), dummy, NULL, NULL); if (!call) return NULL; @@ -4082,7 +4116,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) return NULL; return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s, - decorator_seq, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); + decorator_seq, LINENO(n), n->n_col_offset, + end_lineno, end_col_offset, c->c_arena); } static stmt_ty @@ -4107,7 +4142,8 @@ ast_for_stmt(struct compiling *c, const node *n) case del_stmt: return ast_for_del_stmt(c, n); case pass_stmt: - return Pass(LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Pass(LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); case flow_stmt: return ast_for_flow_stmt(c, n); case import_stmt: From c169025152f080bdfcdb6068a94cf67e5082f1da Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 16 Jan 2019 01:57:28 +0000 Subject: [PATCH 18/36] Fix f-strings --- Python/ast.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index f4206285f1a1e2..9780dcb99f12db 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4400,6 +4400,7 @@ decode_bytes_with_escapes(struct compiling *c, const node *n, const char *s, static void fstring_shift_node_locations(node *n, int lineno, int col_offset) { n->n_col_offset = n->n_col_offset + col_offset; + n->n_end_col_offset = n->n_end_col_offset + col_offset; for (int i = 0; i < NCH(n); ++i) { if (n->n_lineno && n->n_lineno < CHILD(n, i)->n_lineno) { /* Shifting column offsets unnecessary if there's been newlines. */ @@ -4408,6 +4409,7 @@ static void fstring_shift_node_locations(node *n, int lineno, int col_offset) fstring_shift_node_locations(CHILD(n, i), lineno, col_offset); } n->n_lineno = n->n_lineno + lineno; + n->n_end_lineno = n->n_end_lineno + lineno; } /* Fix locations for the given node and its children. @@ -4419,7 +4421,6 @@ static void fstring_shift_node_locations(node *n, int lineno, int col_offset) static void fstring_fix_node_location(const node *parent, node *n, char *expr_str) { - // TODO: fix end positions! char *substr = NULL; char *start; int lines = LINENO(parent) - 1; @@ -5072,7 +5073,8 @@ make_str_node_and_del(PyObject **str, struct compiling *c, const node* n) Py_DECREF(s); return NULL; } - return Constant(s, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(s, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* Add a non-f-string (that is, a regular literal string). str is @@ -5227,7 +5229,8 @@ FstringParser_Finish(FstringParser *state, struct compiling *c, if (!seq) goto error; - return JoinedStr(seq, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return JoinedStr(seq, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); error: FstringParser_Dealloc(state); @@ -5437,7 +5440,8 @@ parsestrplus(struct compiling *c, const node *n) /* Just return the bytes object and we're done. */ if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0) goto error; - return Constant(bytes_str, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); + return Constant(bytes_str, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } /* We're not a bytes string, bytes_str should never have been set. */ From 553a7725633c213a0567e0afdabe38de2318cd00 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 16 Jan 2019 10:50:35 +0000 Subject: [PATCH 19/36] Add few more tests --- Lib/test/test_ast.py | 68 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 16b6ad07df60b8..2e825ca0d8d9fd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1368,7 +1368,7 @@ def test_continued_str(self): self._check_end_pos(assign, 2, 13) self._check_end_pos(assign.value, 2, 13) - def test_control_stmts(self): + def test_suites(self): # We intentionally put these into the same string to check # that empty lines are not part of the suite. s = dedent(''' @@ -1392,6 +1392,72 @@ def test_control_stmts(self): pass ''').strip() + mod = ast.parse(s) + while_loop = mod.body[0] + if_stmt = mod.body[1] + for_loop = mod.body[2] + try_stmt = mod.body[3] + pass_stmt = mod.body[4] + + self._check_end_pos(while_loop, 2, 8) + self._check_end_pos(if_stmt, 9, 12) + self._check_end_pos(for_loop, 12, 15) + self._check_end_pos(try_stmt, 17, 8) + self._check_end_pos(pass_stmt, 19, 4) + + self._check_content(s, while_loop.test, 'True') + self._check_content(s, if_stmt.body[0], 'x = None') + self._check_content(s, if_stmt.orelse[0].test, 'other()') + self._check_content(s, for_loop.target, 'x, y') + self._check_content(s, try_stmt.body[0], 'raise RuntimeError') + self._check_content(s, try_stmt.handlers[0].type, 'TypeError') + + def test_fstring(self): + s = 'x = f"abc {x + y} abc"' + fstr = ast.parse(s).body[0].value + binop = fstr.values[1].value + self._check_content(s, binop, 'x + y') + + def test_fstring_multi_line(self): + s = dedent(''' + f"""Some multi-line text. + { + arg_one + + + arg_two + } + It goes on...""" + ''').strip() + fstr = ast.parse(s).body[0].value + binop = fstr.values[1].value + self._check_end_pos(binop, 5, 7) + self._check_content(s, binop.left, 'arg_one') + self._check_content(s, binop.right, 'arg_two') + + def test_import_from_multi_line(self): + pass + + def test_slices(self): + # test both simple and extended slices, single- and multi-line + pass + + def test_binop(self): + # tricky pars multiline + pass + + def test_boolop(self): + # ditto + pass + + def test_tuples(self): + # commas, spaces, pars + pass + + def test_attribute_spaces(self): + s = 'func(x. y .z)' + call = self._parse_expr(s) + self._check_content(s, call, s) + self._check_content(s, call.args[0], 'x. y .z') def main(): From 5cc01e906c72f0fd171867f6c0f4622f42f6fe8c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 00:53:59 +0000 Subject: [PATCH 20/36] Add final bunch of tests --- Lib/test/test_ast.py | 115 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 2e825ca0d8d9fd..da3450190fab1a 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1291,16 +1291,19 @@ def _check_end_pos(self, ast_node, end_lineno, end_col_offset): def _check_content(self, source, ast_node, content): # should be used for one line expressions only. assert ast_node.lineno == ast_node.end_lineno - self.assertEqual(source.splitlines()[ast_node.lineno - 1] # lines are 1-based + # lines are 1-based + self.assertEqual(source.splitlines()[ast_node.lineno - 1] [ast_node.col_offset:ast_node.end_col_offset], content) - def _parse_expr(self, s): + def _parse_value(self, s): + # Use duck-typing to support both single expression + # and a right hand side of an assignment statement. return ast.parse(s).body[0].value def test_lambda(self): s = 'lambda x, *y: None' - lam = self._parse_expr(s) + lam = self._parse_value(s) self._check_content(s, lam.body, 'None') self._check_content(s, lam.args.args[0], 'x') self._check_content(s, lam.args.vararg, 'y') @@ -1323,14 +1326,14 @@ def func(x: int, def test_call(self): s = 'func(x, y=2, **kw)' - call = self._parse_expr(s) + call = self._parse_value(s) self._check_content(s, call.func, 'func') self._check_content(s, call.keywords[0].value, '2') self._check_content(s, call.keywords[1].value, 'kw') def test_call_noargs(self): s = 'x[0]()' - call = self._parse_expr(s) + call = self._parse_value(s) self._check_content(s, call.func, 'x[0]') self._check_end_pos(call, 1, 6) @@ -1414,7 +1417,7 @@ def test_suites(self): def test_fstring(self): s = 'x = f"abc {x + y} abc"' - fstr = ast.parse(s).body[0].value + fstr = self._parse_value(s) binop = fstr.values[1].value self._check_content(s, binop, 'x + y') @@ -1428,37 +1431,115 @@ def test_fstring_multi_line(self): } It goes on...""" ''').strip() - fstr = ast.parse(s).body[0].value + fstr = self._parse_value(s) binop = fstr.values[1].value self._check_end_pos(binop, 5, 7) self._check_content(s, binop.left, 'arg_one') self._check_content(s, binop.right, 'arg_two') def test_import_from_multi_line(self): - pass + s = dedent(''' + from x.y.z import ( + a, b, c as c + ) + ''').strip() + imp = ast.parse(s).body[0] + self._check_end_pos(imp, 3, 1) def test_slices(self): - # test both simple and extended slices, single- and multi-line - pass + s1 = 'f()[1, 2] [0]' + s2 = 'x[ a.b: c.d]' + sm = dedent(''' + x[ a.b: f () , + g () : c.d + ] + ''').strip() + i1, i2, im = map(self._parse_value, (s1, s2, sm)) + self._check_content(s1, i1.value, 'f()[1, 2]') + self._check_content(s1, i1.value.slice.value, '1, 2') + self._check_content(s2, i2.slice.lower, 'a.b') + self._check_content(s2, i2.slice.upper, 'c.d') + self._check_content(sm, im.slice.dims[0].upper, 'f ()') + self._check_content(sm, im.slice.dims[1].lower, 'g ()') + self._check_end_pos(im, 3, 3) def test_binop(self): - # tricky pars multiline - pass + s = dedent(''' + (1 * 2 + (3 ) + + 4 + ) + ''').strip() + binop = self._parse_value(s) + self._check_end_pos(binop, 2, 6) + self._check_content(s, binop.right, '4') + self._check_content(s, binop.left, '1 * 2 + (3 )') + self._check_content(s, binop.left.right, '3') def test_boolop(self): - # ditto - pass + s = dedent(''' + if (one_condition and + (other_condition or yet_another_one)): + pass + ''').strip() + bop = ast.parse(s).body[0].test + self._check_end_pos(bop, 2, 44) + self._check_content(s, bop.values[1], + 'other_condition or yet_another_one') def test_tuples(self): - # commas, spaces, pars - pass + s1 = 'x = () ;' + s2 = 'x = 1 , ;' + s3 = 'x = (1 , 2 ) ;' + sm = dedent(''' + x = ( + a, b, + ) + ''').strip() + t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) + self._check_content(s1, t1, '()') + self._check_content(s2, t2, '1 ,') + self._check_content(s3, t3, '(1 , 2 )') + self._check_end_pos(tm, 3, 1) def test_attribute_spaces(self): s = 'func(x. y .z)' - call = self._parse_expr(s) + call = self._parse_value(s) self._check_content(s, call, s) self._check_content(s, call.args[0], 'x. y .z') + def test_displays(self): + s1 = '[{}, {1, }, {1, 2,} ]' + s2 = '{a: b, f (): g () ,}' + c1 = self._parse_value(s1) + c2 = self._parse_value(s2) + self._check_content(s1, c1.elts[0], '{}') + self._check_content(s1, c1.elts[1], '{1, }') + self._check_content(s1, c1.elts[2], '{1, 2,}') + self._check_content(s2, c2.keys[1], 'f ()') + self._check_content(s2, c2.values[1], 'g ()') + + def test_comprehensions(self): + s = dedent(''' + x = [{x for x, y in stuff + if cond.x} for stuff in things] + ''').strip() + cmp = self._parse_value(s) + self._check_end_pos(cmp, 2, 37) + self._check_content(s, cmp.generators[0].iter, 'things') + self._check_content(s, cmp.elt.generators[0].iter, 'stuff') + self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') + self._check_content(s, cmp.elt.generators[0].target, 'x, y') + + def test_yield_await(self): + s = dedent(''' + async def f(): + yield x + await y + ''').strip() + fdef = ast.parse(s).body[0] + self._check_content(s, fdef.body[0].value, 'yield x') + self._check_content(s, fdef.body[1].value, 'await y') + def main(): if __name__ != '__main__': From dce260ed7e52034741c6c587c50c6b50b72ab325 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 09:37:17 +0000 Subject: [PATCH 21/36] Update docstrings --- Lib/ast.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 144f310bd212b4..c86a24afee894f 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -115,8 +115,8 @@ def _format(node): def copy_location(new_node, old_node): """ - Copy source location (`lineno` and `col_offset` attributes) from - *old_node* to *new_node* if possible, and return *new_node*. + Copy source location (`lineno`, `col_offset`, `end_lineno`, and `end_col_offset` + attributes) from *old_node* to *new_node* if possible, and return *new_node*. """ for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': if attr in old_node._attributes and attr in new_node._attributes \ @@ -162,8 +162,9 @@ def _fix(node, lineno, col_offset, end_lineno, end_col_offset): def increment_lineno(node, n=1): """ - Increment the line number of each node in the tree starting at *node* by *n*. - This is useful to "move code" to a different location in a file. + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file. """ for child in walk(node): if 'lineno' in child._attributes: From 9ba66049cbb5884769e71415d94dfd86741f5931 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 10:07:56 +0000 Subject: [PATCH 22/36] Update docs --- Doc/library/ast.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 2883f3c6739f16..2ec295952789cf 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -61,13 +61,21 @@ Node classes .. attribute:: lineno col_offset + end_lineno + end_col_offset Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno` and :attr:`col_offset` attributes. The :attr:`lineno` is - the line number of source text (1-indexed so the first line is line 1) and - the :attr:`col_offset` is the UTF-8 byte offset of the first token that - generated the node. The UTF-8 offset is recorded because the parser uses - UTF-8 internally. + :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset` + attributes. The :attr:`lineno` and :attr:`end_lineno` are the first and + last line numbers of source text span (1-indexed so the first line is line 1) + and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding + UTF-8 byte offsets of the first and last tokens that generated the node. + The UTF-8 offset is recorded because the parser uses UTF-8 internally. + + Note that the end positions are not required by the compiler and are + therefore optional. The end offset if *after* the last symbol, for example + one can get the source segment of a one-line expression node using + ``source_line[node.col_offset:node.end_col_offset]``. The constructor of a class :class:`ast.T` parses its arguments as follows: @@ -173,14 +181,16 @@ and classes for traversing abstract syntax trees: .. function:: increment_lineno(node, n=1) - Increment the line number of each node in the tree starting at *node* by *n*. - This is useful to "move code" to a different location in a file. + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file.. .. function:: copy_location(new_node, old_node) - Copy source location (:attr:`lineno` and :attr:`col_offset`) from *old_node* - to *new_node* if possible, and return *new_node*. + Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, + and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, + and return *new_node*. .. function:: iter_fields(node) From 69a6280442be5c1c9a7e20c54e1fe3e3afbda699 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 10:33:18 +0000 Subject: [PATCH 23/36] Add get_source_segment() helper --- Lib/ast.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/ast.py b/Lib/ast.py index c86a24afee894f..521e5312427d03 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -226,6 +226,37 @@ def get_docstring(node, clean=True): return text +def get_source_segment(source, node, coding='utf-8'): + """Get source code segment of the 'source' that generated 'node'. + + If some location information (lineno, end_lineno, col_offset, + or end_col_offset) is missing, return None. The default coding is UTF-8 + (the one used by the Python parser). + """ + if not hasattr(node, 'lineno'): + return None + lineno = node.lineno - 1 + if not hasattr(node, 'end_lineno'): + return None + end_lineno = node.end_lineno - 1 + if not hasattr(node, 'col_offset'): + return None + col_offset = node.col_offset + if not hasattr(node, 'end_col_offset'): + return None + end_col_offset = node.end_col_offset + + if end_lineno == lineno: + source_line = source.splitlines()[lineno] + return source_line.encode(coding)[col_offset:end_col_offset].decode(coding) + + lines = source.splitlines() + first = lines[lineno].encode(coding)[col_offset:].decode(coding) + last = lines[end_lineno].encode(coding)[:end_col_offset].decode(coding) + middle_lines = lines[lineno+1:end_lineno] + return '\n'.join([first] + middle_lines + [last]) + + def walk(node): """ Recursively yield all descendant nodes in the tree starting at *node* From 4af426fdbde8b7f5eecbf7071cea6e61ae8d348b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 10:41:02 +0000 Subject: [PATCH 24/36] Consistent formatting in docstring; use new helper in tests --- Lib/ast.py | 8 ++++---- Lib/test/test_ast.py | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 521e5312427d03..1c6328abbdd4d0 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -227,11 +227,11 @@ def get_docstring(node, clean=True): def get_source_segment(source, node, coding='utf-8'): - """Get source code segment of the 'source' that generated 'node'. + """Get source code segment of the *source* that generated *node*. - If some location information (lineno, end_lineno, col_offset, - or end_col_offset) is missing, return None. The default coding is UTF-8 - (the one used by the Python parser). + If some location information (`lineno`, `end_lineno`, `col_offset`, + or `end_col_offset`) is missing, return None. The default *coding* + is UTF-8 (the one used by the Python parser). """ if not hasattr(node, 'lineno'): return None diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index da3450190fab1a..6f7ebae62b71e2 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1289,12 +1289,7 @@ def _check_end_pos(self, ast_node, end_lineno, end_col_offset): self.assertEqual(ast_node.end_col_offset, end_col_offset) def _check_content(self, source, ast_node, content): - # should be used for one line expressions only. - assert ast_node.lineno == ast_node.end_lineno - # lines are 1-based - self.assertEqual(source.splitlines()[ast_node.lineno - 1] - [ast_node.col_offset:ast_node.end_col_offset], - content) + self.assertEqual(ast.get_source_segment(source, ast_node), content) def _parse_value(self, s): # Use duck-typing to support both single expression From e5a12c30f9b62da25241cfc9a95f906e6a96f1d4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 10:46:58 +0000 Subject: [PATCH 25/36] Fix bug --- Parser/parsetok.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/parsetok.c b/Parser/parsetok.c index ea0cf7c9155c2a..1cbaae10b4db89 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -270,7 +270,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, col_offset = -1; } - if (b != NULL && b >= line_start) { + if (b != NULL && b >= tok->line_start) { end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start, intptr_t, int); } From 0275a93539b41074d4ffcc0999de7fddb2b0cf31 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 17 Jan 2019 11:07:42 +0000 Subject: [PATCH 26/36] Split few long lines --- Python/ast_opt.c | 3 ++- Python/compile.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 6c0298d2b55d50..96c766fc0957d4 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -439,7 +439,8 @@ astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_) return 0; } asdl_seq_SET(values, 0, st->v.Expr.value); - expr_ty expr = JoinedStr(values, st->lineno, st->col_offset, st->end_lineno, st->end_col_offset, ctx_); + expr_ty expr = JoinedStr(values, st->lineno, st->col_offset, + st->end_lineno, st->end_col_offset, ctx_); if (!expr) { return 0; } diff --git a/Python/compile.c b/Python/compile.c index e22e0e41c60664..6a86877f0399f2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4710,7 +4710,8 @@ compiler_augassign(struct compiler *c, stmt_ty s) switch (e->kind) { case Attribute_kind: auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr, - AugLoad, e->lineno, e->col_offset, e->end_lineno, e->end_col_offset, c->c_arena); + AugLoad, e->lineno, e->col_offset + e->end_lineno, e->end_col_offset, c->c_arena); if (auge == NULL) return 0; VISIT(c, expr, auge); @@ -4721,7 +4722,8 @@ compiler_augassign(struct compiler *c, stmt_ty s) break; case Subscript_kind: auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice, - AugLoad, e->lineno, e->col_offset, e->end_lineno, e->end_col_offset, c->c_arena); + AugLoad, e->lineno, e->col_offset, + e->end_lineno, e->end_col_offset, c->c_arena); if (auge == NULL) return 0; VISIT(c, expr, auge); From f20635b255ee6e57b28a4253aed8c13803287435 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 18 Jan 2019 01:02:24 +0000 Subject: [PATCH 27/36] Add tests and docs gor the helper --- Doc/library/ast.rst | 13 +++++++++++++ Lib/ast.py | 18 ++++++++++++++---- Lib/test/test_ast.py | 26 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 2ec295952789cf..f9e26700b86a77 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -170,6 +170,19 @@ and classes for traversing abstract syntax trees: :class:`AsyncFunctionDef` is now supported. +.. function:: get_code_segment(source, node, *, padded=False, coding='utf-8') + + Get source code segment of the *source* that generated *node*. + If some location information (:attr:`lineno`, :attr:`end_lineno`, + :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. + The default *coding* is UTF-8 (the one used by the Python parser). + + If *padded* is ``True``, the first line of a multi-line statement will + be padded with spaces to match its original position. + + .. versionadded:: 3.8 + + .. function:: fix_missing_locations(node) When you compile a node tree with :func:`compile`, the compiler expects diff --git a/Lib/ast.py b/Lib/ast.py index 1c6328abbdd4d0..8e07ee3d399922 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -226,12 +226,15 @@ def get_docstring(node, clean=True): return text -def get_source_segment(source, node, coding='utf-8'): +def get_source_segment(source, node, *, padded=False, coding='utf-8'): """Get source code segment of the *source* that generated *node*. If some location information (`lineno`, `end_lineno`, `col_offset`, or `end_col_offset`) is missing, return None. The default *coding* is UTF-8 (the one used by the Python parser). + + If *padded* is `True`, the first line of a multi-line statement will + be padded with spaces to match its original position. """ if not hasattr(node, 'lineno'): return None @@ -248,12 +251,19 @@ def get_source_segment(source, node, coding='utf-8'): if end_lineno == lineno: source_line = source.splitlines()[lineno] - return source_line.encode(coding)[col_offset:end_col_offset].decode(coding) + return source_line.encode()[col_offset:end_col_offset].decode(coding) lines = source.splitlines() - first = lines[lineno].encode(coding)[col_offset:].decode(coding) - last = lines[end_lineno].encode(coding)[:end_col_offset].decode(coding) + + if padded: + padding = ' ' * len(lines[lineno].encode()[:col_offset].decode(coding)) + else: + padding = '' + + first = padding + lines[lineno].encode()[col_offset:].decode(coding) + last = lines[end_lineno].encode()[:end_col_offset].decode(coding) middle_lines = lines[lineno+1:end_lineno] + return '\n'.join([first] + middle_lines + [last]) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 6f7ebae62b71e2..12832cb056dac2 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1535,6 +1535,32 @@ async def f(): self._check_content(s, fdef.body[0].value, 'yield x') self._check_content(s, fdef.body[1].value, 'await y') + def test_source_segment_multi(self): + s_orig = dedent(''' + x = ( + a, b, + ) + () + ''').strip() + s_tuple = dedent(''' + ( + a, b, + ) + ''').strip() + binop = self._parse_value(s_orig) + self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) + + def test_source_segment_padded(self): + s_orig = dedent(''' + class C: + def fun(self) -> None: + "ЖЖЖЖЖ" + ''').strip() + s_method = ' def fun(self) -> None:\n' \ + ' "ЖЖЖЖЖ"' + cdef = ast.parse(s_orig).body[0] + self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), + s_method) + def main(): if __name__ != '__main__': From c9da8f54c45e3dff3308acffcf07964aad025564 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 18 Jan 2019 01:04:19 +0000 Subject: [PATCH 28/36] Fix missing comma --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index 6a86877f0399f2..1ab9908bee9e9b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4710,7 +4710,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) switch (e->kind) { case Attribute_kind: auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr, - AugLoad, e->lineno, e->col_offset + AugLoad, e->lineno, e->col_offset, e->end_lineno, e->end_col_offset, c->c_arena); if (auge == NULL) return 0; From f97f38a111382730da82200dfd35e5f3a5236027 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" Date: Fri, 18 Jan 2019 09:49:17 +0000 Subject: [PATCH 29/36] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst b/Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst new file mode 100644 index 00000000000000..dd328824af91da --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst @@ -0,0 +1 @@ +Add end line and end column position information to the Python AST nodes. \ No newline at end of file From 70cc16c7bf767f9b77cb0846f12da2a3a34de033 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 18 Jan 2019 10:38:53 +0000 Subject: [PATCH 30/36] Fix .rst warning and a smelly symbol --- Doc/library/ast.rst | 2 +- Include/node.h | 2 +- Parser/node.c | 6 +++--- Parser/parsetok.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index f9e26700b86a77..de35db68d98061 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -75,7 +75,7 @@ Node classes Note that the end positions are not required by the compiler and are therefore optional. The end offset if *after* the last symbol, for example one can get the source segment of a one-line expression node using - ``source_line[node.col_offset:node.end_col_offset]``. + ``source_line[node.col_offset : node.end_col_offset]``. The constructor of a class :class:`ast.T` parses its arguments as follows: diff --git a/Include/node.h b/Include/node.h index b2752b2bd6b070..2b390740973873 100644 --- a/Include/node.h +++ b/Include/node.h @@ -40,7 +40,7 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); #define REQ(n, type) assert(TYPE(n) == (type)) PyAPI_FUNC(void) PyNode_ListTree(node *); -void _finalize_end_pos(node *n); // helper also used in parsetok.c +void _PyNode_FinalizeEndPos(node *n); // helper also used in parsetok.c #ifdef __cplusplus } diff --git a/Parser/node.c b/Parser/node.c index 160a97b917391a..f1b70e0f6815be 100644 --- a/Parser/node.c +++ b/Parser/node.c @@ -78,7 +78,7 @@ fancy_roundup(int n) void -_finalize_end_pos(node *n) +_PyNode_FinalizeEndPos(node *n) { int nch = NCH(n); node *last; @@ -86,7 +86,7 @@ _finalize_end_pos(node *n) return; } last = CHILD(n, nch - 1); - _finalize_end_pos(last); + _PyNode_FinalizeEndPos(last); n->n_end_lineno = last->n_end_lineno; n->n_end_col_offset = last->n_end_col_offset; } @@ -102,7 +102,7 @@ PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset, // finalize end position of previous node (if any) if (nch > 0) { - _finalize_end_pos(CHILD(n1, nch - 1)); + _PyNode_FinalizeEndPos(CHILD(n1, nch - 1)); } if (nch == INT_MAX || nch < 0) diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 1cbaae10b4db89..2b5254a8be67ad 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -376,7 +376,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, PyTokenizer_Free(tok); if (n != NULL) { - _finalize_end_pos(n); + _PyNode_FinalizeEndPos(n); } return n; } From 48936b9b3f7205e036bc4676f220fde850a91f79 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 19 Jan 2019 17:26:53 +0000 Subject: [PATCH 31/36] Fix get_source_segment --- Lib/ast.py | 76 ++++++++++++++++++++++++++++++-------------- Lib/test/test_ast.py | 21 ++++++++++++ 2 files changed, 74 insertions(+), 23 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 8e07ee3d399922..5e62ffc86e7872 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -226,45 +226,75 @@ def get_docstring(node, clean=True): return text -def get_source_segment(source, node, *, padded=False, coding='utf-8'): +def _splitlines_no_ff(source): + """Split a string into lines ignoring form feed and other chars. + + This mimics how the Python parser splits source code. + """ + idx = 0 + lines = [] + next_line = '' + while idx < len(source): + c = source[idx] + next_line += c + idx += 1 + # Keep \r\n together + if c == '\r' and idx < len(source) and source[idx] == '\n': + next_line += '\n' + idx += 1 + if c in '\r\n': + lines.append(next_line) + next_line = '' + + if next_line: + lines.append(next_line) + return lines + + +def _pad_whitespace(source): + """Replace all chars except '\f\t' in a line with spaces.""" + result = '' + for c in source: + if c in '\f\t': + result += c + else: + result += ' ' + return result + + +def get_source_segment(source, node, *, padded=False): """Get source code segment of the *source* that generated *node*. If some location information (`lineno`, `end_lineno`, `col_offset`, - or `end_col_offset`) is missing, return None. The default *coding* - is UTF-8 (the one used by the Python parser). + or `end_col_offset`) is missing, return None. If *padded* is `True`, the first line of a multi-line statement will be padded with spaces to match its original position. """ - if not hasattr(node, 'lineno'): - return None - lineno = node.lineno - 1 - if not hasattr(node, 'end_lineno'): + try: + lineno = node.lineno - 1 + end_lineno = node.end_lineno - 1 + col_offset = node.col_offset + end_col_offset = node.end_col_offset + except AttributeError: return None - end_lineno = node.end_lineno - 1 - if not hasattr(node, 'col_offset'): - return None - col_offset = node.col_offset - if not hasattr(node, 'end_col_offset'): - return None - end_col_offset = node.end_col_offset + lines = _splitlines_no_ff(source) if end_lineno == lineno: - source_line = source.splitlines()[lineno] - return source_line.encode()[col_offset:end_col_offset].decode(coding) - - lines = source.splitlines() + return lines[lineno].encode()[col_offset:end_col_offset].decode() if padded: - padding = ' ' * len(lines[lineno].encode()[:col_offset].decode(coding)) + padding = _pad_whitespace(lines[lineno].encode()[:col_offset].decode()) else: padding = '' - first = padding + lines[lineno].encode()[col_offset:].decode(coding) - last = lines[end_lineno].encode()[:end_col_offset].decode(coding) - middle_lines = lines[lineno+1:end_lineno] + first = padding + lines[lineno].encode()[col_offset:].decode() + last = lines[end_lineno].encode()[:end_col_offset].decode() + lines = lines[lineno+1:end_lineno] - return '\n'.join([first] + middle_lines + [last]) + lines.insert(0, first) + lines.append(last) + return ''.join(lines) def walk(node): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 12832cb056dac2..cdc092bb84188e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1561,6 +1561,27 @@ def fun(self) -> None: self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), s_method) + def test_source_segment_endings(self): + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' + v, w, x, y, z = ast.parse(s).body + self._check_content(s, v, 'v = 1') + self._check_content(s, w, 'w = 1') + self._check_content(s, x, 'x = 1') + self._check_content(s, y, 'y = 1') + self._check_content(s, z, 'z = 1') + + def test_source_segment_tabs(self): + s = dedent(''' + class C: + \t\f def fun(self) -> None: + \t\f pass + ''').strip() + s_method = ' \t\f def fun(self) -> None:\n' \ + ' \t\f pass' + + cdef = ast.parse(s).body[0] + self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) + def main(): if __name__ != '__main__': From ac5b5cb7617180306f1668b0ce73ea2a8dd56c6a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 19 Jan 2019 19:10:47 +0000 Subject: [PATCH 32/36] More CR --- Doc/library/ast.rst | 3 +-- Include/Python-ast.h | 1 + Lib/test/test_asdl_parser.py | 2 ++ Parser/asdl_c.py | 8 +++++--- Python/ast.c | 13 +++++++------ 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index de35db68d98061..16e4a65e5968af 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -170,12 +170,11 @@ and classes for traversing abstract syntax trees: :class:`AsyncFunctionDef` is now supported. -.. function:: get_code_segment(source, node, *, padded=False, coding='utf-8') +.. function:: get_source_segment(source, node, *, padded=False) Get source code segment of the *source* that generated *node*. If some location information (:attr:`lineno`, :attr:`end_lineno`, :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. - The default *coding* is UTF-8 (the one used by the Python parser). If *padded* is ``True``, the first line of a multi-line statement will be padded with spaces to match its original position. diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 9d290966cfd89e..f8394e6c26ad83 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -438,6 +438,7 @@ struct _withitem { }; +// Note: these macros affect function definitions, not only call sites. #define Module(a0, a1) _Py_Module(a0, a1) mod_ty _Py_Module(asdl_seq * body, PyArena *arena); #define Interactive(a0, a1) _Py_Interactive(a0, a1) diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py index e342ece0731412..30e6466dbb325a 100644 --- a/Lib/test/test_asdl_parser.py +++ b/Lib/test/test_asdl_parser.py @@ -65,6 +65,8 @@ def test_attributes(self): self.assertEqual(len(stmt.attributes), 4) self.assertEqual(str(stmt.attributes[0]), 'Field(int, lineno)') self.assertEqual(str(stmt.attributes[1]), 'Field(int, col_offset)') + self.assertEqual(str(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)') + self.assertEqual(str(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)') def test_constructor_fields(self): ehandler = self.types['excepthandler'] diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 75fb78b9c94e15..8640b29b8f10b6 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1250,10 +1250,12 @@ def main(srcfile, dump_module=False): f.write('#undef Yield /* undefine macro conflicting with */\n') f.write('\n') c = ChainOfVisitors(TypeDefVisitor(f), - StructVisitor(f), - PrototypeVisitor(f), - ) + StructVisitor(f)) + c.visit(mod) + f.write("// Note: these macros affect function definitions, not only call sites.\n") + PrototypeVisitor(f).visit(mod) + f.write("\n") f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") f.write("int PyAST_Check(PyObject* obj);\n") diff --git a/Python/ast.c b/Python/ast.c index 9780dcb99f12db..e799ed050f0898 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2918,12 +2918,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, } } - // Ideally call ends where closing parenthesis is placed. - end_lineno = closepar == NULL ? n->n_end_lineno : closepar->n_end_lineno; - end_col_offset = closepar == NULL ? n->n_end_col_offset : closepar->n_end_col_offset; - return Call(func, args, keywords, func->lineno, func->col_offset, - end_lineno, end_col_offset, c->c_arena); + closepar->n_end_lineno, closepar->n_end_col_offset, c->c_arena); } static expr_ty @@ -3612,6 +3608,11 @@ static void get_last_end_pos(asdl_seq *s, int *end_lineno, int *end_col_offset) { int tot = asdl_seq_LEN(s); + // Suite should not be empty, but it is safe to just ignore it + // if it will ever occur. + if (!tot) { + return; + } stmt_ty last = asdl_seq_GET(s, tot - 1); *end_lineno = last->end_lineno; *end_col_offset = last->end_col_offset; @@ -4100,7 +4101,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, c->c_arena); - call = ast_for_call(c, CHILD(n, 3), dummy, NULL, NULL); + call = ast_for_call(c, CHILD(n, 3), dummy, NULL, CHILD(n, 4)); if (!call) return NULL; } From 4726f17bfe0dce24e9671d0f14bebe7b2f7188fd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 19 Jan 2019 19:11:53 +0000 Subject: [PATCH 33/36] rst fixes --- Doc/library/ast.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 16e4a65e5968af..7715a28ce1b873 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -73,7 +73,7 @@ Node classes The UTF-8 offset is recorded because the parser uses UTF-8 internally. Note that the end positions are not required by the compiler and are - therefore optional. The end offset if *after* the last symbol, for example + therefore optional. The end offset is *after* the last symbol, for example one can get the source segment of a one-line expression node using ``source_line[node.col_offset : node.end_col_offset]``. @@ -195,7 +195,7 @@ and classes for traversing abstract syntax trees: Increment the line number and end line number of each node in the tree starting at *node* by *n*. This is useful to "move code" to a different - location in a file.. + location in a file. .. function:: copy_location(new_node, old_node) From eeea87d87f4d58649f7faee9bb75ec05268fa074 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 19 Jan 2019 19:17:51 +0000 Subject: [PATCH 34/36] Remove unused vars --- Python/ast.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index e799ed050f0898..855acca29e7c3c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2745,7 +2745,6 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, int i, nargs, nkeywords; int ndoublestars; - int end_lineno, end_col_offset; asdl_seq *args; asdl_seq *keywords; From 027a4cae21cf8ce25ea8c9168b856ee37b038099 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" Date: Sat, 19 Jan 2019 19:41:54 +0000 Subject: [PATCH 35/36] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst b/Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst new file mode 100644 index 00000000000000..2e618e4f0e1f25 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst @@ -0,0 +1,2 @@ +Add end line and end column position information to the Python AST nodes. +This is a C-level backwards incompatible change. \ No newline at end of file From ff361f26b3aabb8c27706b399f5d03f922d7ea61 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 19 Jan 2019 19:47:26 +0000 Subject: [PATCH 36/36] Remove old NEWS file --- .../Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst b/Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst deleted file mode 100644 index dd328824af91da..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-01-18-09-49-17.bpo-33416.rrj9a5.rst +++ /dev/null @@ -1 +0,0 @@ -Add end line and end column position information to the Python AST nodes. \ No newline at end of file