From 290280b62766a1f6d133c998a859dc3f65dee8c2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 13:21:46 +0200 Subject: [PATCH 01/13] Use the traceback module for PyErr_Display() and fallback to the C implementation --- Lib/traceback.py | 26 ++- Python/pythonrun.c | 533 ++++----------------------------------------- Python/traceback.c | 252 +-------------------- 3 files changed, 76 insertions(+), 735 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 67941ff45988c2..2adce608442268 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -125,6 +125,14 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ te.print(file=file, chain=chain) +BUILTIN_EXCEPTION_LIMIT = object() + + +def _print_exception_bltin(exc, /): + file = sys.stderr if sys.stderr is not None else sys.__stderr__ + return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file) + + def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ chain=True): """Format a stack trace and the exception information. @@ -406,12 +414,16 @@ def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, # (frame, (lineno, end_lineno, colno, end_colno)) in the stack. # Only lineno is required, the remaining fields can be None if the # information is not available. - if limit is None: + builtin_limit = limit is BUILTIN_EXCEPTION_LIMIT + if limit is None or limit is BUILTIN_EXCEPTION_LIMIT: limit = getattr(sys, 'tracebacklimit', None) if limit is not None and limit < 0: limit = 0 if limit is not None: - if limit >= 0: + if builtin_limit: + frame_gen = tuple(frame_gen) + frame_gen = frame_gen[len(frame_gen) - limit:] + elif limit >= 0: frame_gen = itertools.islice(frame_gen, limit) else: frame_gen = collections.deque(frame_gen, maxlen=-limit) @@ -741,9 +753,9 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, wrong_name = getattr(exc_value, "name", None) if wrong_name is not None and wrong_name in sys.stdlib_module_names: if suggestion: - self._str += f" Or did you forget to import '{wrong_name}'" + self._str += f" Or did you forget to import '{wrong_name}'?" else: - self._str += f". Did you forget to import '{wrong_name}'" + self._str += f". Did you forget to import '{wrong_name}'?" if lookup_lines: self._load_lines() self.__suppress_context__ = \ @@ -904,7 +916,11 @@ def _format_syntax_error(self, stype): if self.offset is not None: offset = self.offset end_offset = self.end_offset if self.end_offset not in {None, 0} else offset - if offset == end_offset or end_offset == -1: + if self.text and offset > len(self.text): + offset = len(self.text) + 1 + if self.text and end_offset > len(self.text): + end_offset = len(self.text) + 1 + if offset >= end_offset or end_offset < 0: end_offset = offset + 1 # Convert 1-based column offset to 0-based index into stripped text diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 1b282aa870bfd2..029a07d511ac50 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -517,204 +517,6 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return 0; } -static int -parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, - Py_ssize_t *lineno, Py_ssize_t *offset, - Py_ssize_t* end_lineno, Py_ssize_t* end_offset, - PyObject **text) -{ - Py_ssize_t hold; - PyObject *v; - - *message = NULL; - *filename = NULL; - - /* new style errors. `err' is an instance */ - *message = PyObject_GetAttr(err, &_Py_ID(msg)); - if (!*message) - goto finally; - - v = PyObject_GetAttr(err, &_Py_ID(filename)); - if (!v) - goto finally; - if (v == Py_None) { - Py_DECREF(v); - _Py_DECLARE_STR(anon_string, ""); - *filename = Py_NewRef(&_Py_STR(anon_string)); - } - else { - *filename = v; - } - - v = PyObject_GetAttr(err, &_Py_ID(lineno)); - if (!v) - goto finally; - hold = PyLong_AsSsize_t(v); - Py_DECREF(v); - if (hold < 0 && PyErr_Occurred()) - goto finally; - *lineno = hold; - - v = PyObject_GetAttr(err, &_Py_ID(offset)); - if (!v) - goto finally; - if (v == Py_None) { - *offset = -1; - Py_DECREF(v); - } else { - hold = PyLong_AsSsize_t(v); - Py_DECREF(v); - if (hold < 0 && PyErr_Occurred()) - goto finally; - *offset = hold; - } - - if (Py_TYPE(err) == (PyTypeObject*)PyExc_SyntaxError) { - v = PyObject_GetAttr(err, &_Py_ID(end_lineno)); - if (!v) { - PyErr_Clear(); - *end_lineno = *lineno; - } - else if (v == Py_None) { - *end_lineno = *lineno; - Py_DECREF(v); - } else { - hold = PyLong_AsSsize_t(v); - Py_DECREF(v); - if (hold < 0 && PyErr_Occurred()) - goto finally; - *end_lineno = hold; - } - - v = PyObject_GetAttr(err, &_Py_ID(end_offset)); - if (!v) { - PyErr_Clear(); - *end_offset = -1; - } - else if (v == Py_None) { - *end_offset = -1; - Py_DECREF(v); - } else { - hold = PyLong_AsSsize_t(v); - Py_DECREF(v); - if (hold < 0 && PyErr_Occurred()) - goto finally; - *end_offset = hold; - } - } else { - // SyntaxError subclasses - *end_lineno = *lineno; - *end_offset = -1; - } - - v = PyObject_GetAttr(err, &_Py_ID(text)); - if (!v) - goto finally; - if (v == Py_None) { - Py_DECREF(v); - *text = NULL; - } - else { - *text = v; - } - return 1; - -finally: - Py_XDECREF(*message); - Py_XDECREF(*filename); - return 0; -} - -static int -print_error_text(PyObject *f, Py_ssize_t offset, Py_ssize_t end_offset, - PyObject *text_obj) -{ - size_t caret_repetitions = (end_offset > 0 && end_offset > offset) ? - end_offset - offset : 1; - - /* Convert text to a char pointer; return if error */ - const char *text = PyUnicode_AsUTF8(text_obj); - if (text == NULL) { - return -1; - } - - /* Convert offset from 1-based to 0-based */ - offset--; - - /* Strip leading whitespace from text, adjusting offset as we go */ - while (*text == ' ' || *text == '\t' || *text == '\f') { - text++; - offset--; - } - - /* Calculate text length excluding trailing newline */ - Py_ssize_t len = strlen(text); - if (len > 0 && text[len-1] == '\n') { - len--; - } - - /* Clip offset to at most len */ - if (offset > len) { - offset = len; - } - - /* Skip past newlines embedded in text */ - for (;;) { - const char *nl = strchr(text, '\n'); - if (nl == NULL) { - break; - } - Py_ssize_t inl = nl - text; - if (inl >= offset) { - break; - } - inl += 1; - text += inl; - len -= inl; - offset -= (int)inl; - } - - /* Print text */ - if (PyFile_WriteString(" ", f) < 0) { - return -1; - } - if (PyFile_WriteString(text, f) < 0) { - return -1; - } - - /* Make sure there's a newline at the end */ - if (text[len] != '\n') { - if (PyFile_WriteString("\n", f) < 0) { - return -1; - } - } - - /* Don't print caret if it points to the left of the text */ - if (offset < 0) { - return 0; - } - - /* Write caret line */ - if (PyFile_WriteString(" ", f) < 0) { - return -1; - } - while (--offset >= 0) { - if (PyFile_WriteString(" ", f) < 0) { - return -1; - } - } - for (size_t caret_iter=0; caret_iter < caret_repetitions ; caret_iter++) { - if (PyFile_WriteString("^", f) < 0) { - return -1; - } - } - if (PyFile_WriteString("\n", f) < 0) { - return -1; - } - return 0; -} - - int _Py_HandleSystemExit(int *exitcode_p) { @@ -883,14 +685,10 @@ struct exception_print_context { PyObject *file; PyObject *seen; // Prevent cycles in recursion - int exception_group_depth; // nesting level of current exception group bool need_close; // Need a closing bottom frame - int max_group_width; // Maximum number of children of each EG - int max_group_depth; // Maximum nesting level of EGs }; - -#define EXC_MARGIN(ctx) ((ctx)->exception_group_depth ? "| " : "") -#define EXC_INDENT(ctx) (2 * (ctx)->exception_group_depth) +#define EXC_MARGIN(ctx) ("") +#define EXC_INDENT(ctx) (0) static int write_indented_margin(struct exception_print_context *ctx, PyObject *f) @@ -930,12 +728,6 @@ print_exception_traceback(struct exception_print_context *ctx, PyObject *value) if (tb && tb != Py_None) { const char *header = EXCEPTION_TB_HEADER; const char *header_margin = EXC_MARGIN(ctx); - if (_PyBaseExceptionGroup_Check(value)) { - header = EXCEPTION_GROUP_TB_HEADER; - if (ctx->exception_group_depth == 1) { - header_margin = "+ "; - } - } err = _PyTraceBack_Print_Indented( tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), header_margin, header, f); } @@ -959,16 +751,20 @@ print_exception_file_and_line(struct exception_print_context *ctx, } Py_DECREF(tmp); - PyObject *message, *filename, *text; - Py_ssize_t lineno, offset, end_lineno, end_offset; - if (!parse_syntax_error(*value_p, &message, &filename, - &lineno, &offset, - &end_lineno, &end_offset, &text)) { - PyErr_Clear(); - return 0; + PyObject *filename = NULL; + Py_ssize_t lineno = 0; + PyObject* v = PyObject_GetAttr(*value_p, &_Py_ID(filename)); + if (!v) { + return -1; + } + if (v == Py_None) { + Py_DECREF(v); + _Py_DECLARE_STR(anon_string, ""); + filename = Py_NewRef(&_Py_STR(anon_string)); + } + else { + filename = v; } - - Py_SETREF(*value_p, message); PyObject *line = PyUnicode_FromFormat(" File \"%S\", line %zd\n", filename, lineno); @@ -984,32 +780,11 @@ print_exception_file_and_line(struct exception_print_context *ctx, } Py_CLEAR(line); - if (text != NULL) { - Py_ssize_t line_size; - const char *error_line = PyUnicode_AsUTF8AndSize(text, &line_size); - // If the location of the error spawn multiple lines, we want - // to just print the first one and highlight everything until - // the end of that one since we don't support multi-line error - // messages. - if (end_lineno > lineno) { - end_offset = (error_line != NULL) ? line_size : -1; - } - // Limit the amount of '^' that we can display to - // the size of the text in the source line. - if (error_line != NULL && end_offset > line_size + 1) { - end_offset = line_size + 1; - } - if (print_error_text(f, offset, end_offset, text) < 0) { - goto error; - } - Py_DECREF(text); - } assert(!PyErr_Occurred()); return 0; error: Py_XDECREF(line); - Py_XDECREF(text); return -1; } @@ -1098,104 +873,9 @@ print_exception_message(struct exception_print_context *ctx, PyObject *type, return 0; } -static int -print_exception_suggestions(struct exception_print_context *ctx, - PyObject *value) -{ - PyObject *f = ctx->file; - PyObject *suggestions = _Py_Offer_Suggestions(value); - if (suggestions) { - if (PyFile_WriteObject(suggestions, f, Py_PRINT_RAW) < 0) { - goto error; - } - Py_DECREF(suggestions); - } - else if (PyErr_Occurred()) { - PyErr_Clear(); - } - return 0; -error: - Py_XDECREF(suggestions); - return -1; -} - -static int -print_exception_notes(struct exception_print_context *ctx, PyObject *notes) -{ - PyObject *f = ctx->file; - - if (notes == NULL) { - return 0; - } - - if (!PySequence_Check(notes) || PyUnicode_Check(notes) || PyBytes_Check(notes)) { - int res = 0; - if (write_indented_margin(ctx, f) < 0) { - res = -1; - } - PyObject *s = PyObject_Repr(notes); - if (s == NULL) { - PyErr_Clear(); - res = PyFile_WriteString("<__notes__ repr() failed>", f); - } - else { - res = PyFile_WriteObject(s, f, Py_PRINT_RAW); - Py_DECREF(s); - } - if (PyFile_WriteString("\n", f) < 0) { - res = -1; - } - return res; - } - Py_ssize_t num_notes = PySequence_Length(notes); - PyObject *lines = NULL; - for (Py_ssize_t ni = 0; ni < num_notes; ni++) { - PyObject *note = PySequence_GetItem(notes, ni); - PyObject *note_str = PyObject_Str(note); - Py_DECREF(note); - - if (note_str == NULL) { - PyErr_Clear(); - if (PyFile_WriteString("", f) < 0) { - goto error; - } - } - else { - lines = PyUnicode_Splitlines(note_str, 1); - Py_DECREF(note_str); - - if (lines == NULL) { - goto error; - } - - Py_ssize_t n = PyList_GET_SIZE(lines); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *line = PyList_GET_ITEM(lines, i); - assert(PyUnicode_Check(line)); - if (write_indented_margin(ctx, f) < 0) { - goto error; - } - if (PyFile_WriteObject(line, f, Py_PRINT_RAW) < 0) { - goto error; - } - } - Py_CLEAR(lines); - } - if (PyFile_WriteString("\n", f) < 0) { - goto error; - } - } - - return 0; -error: - Py_XDECREF(lines); - return -1; -} - static int print_exception(struct exception_print_context *ctx, PyObject *value) { - PyObject *notes = NULL; PyObject *f = ctx->file; if (!PyExceptionInstance_Check(value)) { @@ -1211,9 +891,6 @@ print_exception(struct exception_print_context *ctx, PyObject *value) /* grab the type and notes now because value can change below */ PyObject *type = (PyObject *) Py_TYPE(value); - if (PyObject_GetOptionalAttr(value, &_Py_ID(__notes__), ¬es) < 0) { - goto error; - } if (print_exception_file_and_line(ctx, &value) < 0) { goto error; @@ -1221,22 +898,13 @@ print_exception(struct exception_print_context *ctx, PyObject *value) if (print_exception_message(ctx, type, value) < 0) { goto error; } - if (print_exception_suggestions(ctx, value) < 0) { - goto error; - } if (PyFile_WriteString("\n", f) < 0) { goto error; } - if (print_exception_notes(ctx, notes) < 0) { - goto error; - } - - Py_XDECREF(notes); Py_DECREF(value); assert(!PyErr_Occurred()); return 0; error: - Py_XDECREF(notes); Py_DECREF(value); return -1; } @@ -1359,133 +1027,6 @@ print_exception_cause_and_context(struct exception_print_context *ctx, return 0; } -static int -print_exception_group(struct exception_print_context *ctx, PyObject *value) -{ - PyObject *f = ctx->file; - - if (ctx->exception_group_depth > ctx->max_group_depth) { - /* depth exceeds limit */ - - if (write_indented_margin(ctx, f) < 0) { - return -1; - } - - PyObject *line = PyUnicode_FromFormat("... (max_group_depth is %d)\n", - ctx->max_group_depth); - if (line == NULL) { - return -1; - } - int err = PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_DECREF(line); - return err; - } - - if (ctx->exception_group_depth == 0) { - ctx->exception_group_depth += 1; - } - if (print_exception(ctx, value) < 0) { - return -1; - } - - PyObject *excs = ((PyBaseExceptionGroupObject *)value)->excs; - assert(excs && PyTuple_Check(excs)); - Py_ssize_t num_excs = PyTuple_GET_SIZE(excs); - assert(num_excs > 0); - Py_ssize_t n; - if (num_excs <= ctx->max_group_width) { - n = num_excs; - } - else { - n = ctx->max_group_width + 1; - } - - ctx->need_close = false; - for (Py_ssize_t i = 0; i < n; i++) { - bool last_exc = (i == n - 1); - if (last_exc) { - // The closing frame may be added in a recursive call - ctx->need_close = true; - } - - if (_Py_WriteIndent(EXC_INDENT(ctx), f) < 0) { - return -1; - } - bool truncated = (i >= ctx->max_group_width); - PyObject *line; - if (!truncated) { - line = PyUnicode_FromFormat( - "%s+---------------- %zd ----------------\n", - (i == 0) ? "+-" : " ", i + 1); - } - else { - line = PyUnicode_FromFormat( - "%s+---------------- ... ----------------\n", - (i == 0) ? "+-" : " "); - } - if (line == NULL) { - return -1; - } - int err = PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_DECREF(line); - if (err < 0) { - return -1; - } - - ctx->exception_group_depth += 1; - PyObject *exc = PyTuple_GET_ITEM(excs, i); - - if (!truncated) { - if (_Py_EnterRecursiveCall(" in print_exception_group")) { - return -1; - } - int res = print_exception_recursive(ctx, exc); - _Py_LeaveRecursiveCall(); - if (res < 0) { - return -1; - } - } - else { - Py_ssize_t excs_remaining = num_excs - ctx->max_group_width; - - if (write_indented_margin(ctx, f) < 0) { - return -1; - } - - PyObject *line = PyUnicode_FromFormat( - "and %zd more exception%s\n", - excs_remaining, excs_remaining > 1 ? "s" : ""); - - if (line == NULL) { - return -1; - } - - int err = PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_DECREF(line); - if (err < 0) { - return -1; - } - } - - if (last_exc && ctx->need_close) { - if (_Py_WriteIndent(EXC_INDENT(ctx), f) < 0) { - return -1; - } - if (PyFile_WriteString( - "+------------------------------------\n", f) < 0) { - return -1; - } - ctx->need_close = false; - } - ctx->exception_group_depth -= 1; - } - - if (ctx->exception_group_depth == 1) { - ctx->exception_group_depth -= 1; - } - return 0; -} - static int print_exception_recursive(struct exception_print_context *ctx, PyObject *value) { @@ -1498,12 +1039,7 @@ print_exception_recursive(struct exception_print_context *ctx, PyObject *value) goto error; } } - if (!_PyBaseExceptionGroup_Check(value)) { - if (print_exception(ctx, value) < 0) { - goto error; - } - } - else if (print_exception_group(ctx, value) < 0) { + if (print_exception(ctx, value) < 0) { goto error; } assert(!PyErr_Occurred()); @@ -1515,9 +1051,6 @@ print_exception_recursive(struct exception_print_context *ctx, PyObject *value) return -1; } -#define PyErr_MAX_GROUP_WIDTH 15 -#define PyErr_MAX_GROUP_DEPTH 10 - void _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) { @@ -1535,12 +1068,40 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } } + if (!value) { + goto fallback; + } + + // Try first with the traceback. + int unhandled_keyboard_interrupt = _PyRuntime.signals.unhandled_keyboard_interrupt; + PyObject *traceback_module = PyImport_ImportModule("traceback"); + + if (traceback_module == NULL) { + goto fallback; + } + + PyObject *print_tb_func = PyObject_GetAttrString(traceback_module, "_print_exception_bltin"); + + if (print_tb_func == NULL || !PyCallable_Check(print_tb_func)) { + Py_DECREF(traceback_module); + goto fallback; + } + + PyObject* result = PyObject_CallOneArg(print_tb_func, value); + + Py_DECREF(traceback_module); + Py_XDECREF(print_tb_func); + _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt; + if (result) { + Py_DECREF(result); + return; + } +fallback: + PyErr_Clear(); + struct exception_print_context ctx; ctx.file = file; - ctx.exception_group_depth = 0; ctx.need_close = false; - ctx.max_group_width = PyErr_MAX_GROUP_WIDTH; - ctx.max_group_depth = PyErr_MAX_GROUP_DEPTH; /* We choose to ignore seen being possibly NULL, and report at least the main exception (it could be a MemoryError). diff --git a/Python/traceback.c b/Python/traceback.c index 5de1bff9943c6c..0f995a0df700a5 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -578,157 +578,8 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, NULL, truncation, line); } -/* AST based Traceback Specialization - * - * When displaying a new traceback line, for certain syntactical constructs - * (e.g a subscript, an arithmetic operation) we try to create a representation - * that separates the primary source of error from the rest. - * - * Example specialization of BinOp nodes: - * Traceback (most recent call last): - * File "/home/isidentical/cpython/cpython/t.py", line 10, in - * add_values(1, 2, 'x', 3, 4) - * File "/home/isidentical/cpython/cpython/t.py", line 2, in add_values - * return a + b + c + d + e - * ~~~~~~^~~ - * TypeError: 'NoneType' object is not subscriptable - */ #define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\f')) - -static int -extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *left_anchor, Py_ssize_t *right_anchor, - char** primary_error_char, char** secondary_error_char) -{ - switch (expr->kind) { - case BinOp_kind: { - expr_ty left = expr->v.BinOp.left; - expr_ty right = expr->v.BinOp.right; - for (int i = left->end_col_offset; i < right->col_offset; i++) { - if (IS_WHITESPACE(segment_str[i])) { - continue; - } - - *left_anchor = i; - *right_anchor = i + 1; - - // Check whether if this a two-character operator (e.g //) - if (i + 1 < right->col_offset && !IS_WHITESPACE(segment_str[i + 1])) { - ++*right_anchor; - } - - // Keep going if the current char is not ')' - if (i+1 < right->col_offset && (segment_str[i] == ')')) { - continue; - } - - // Set the error characters - *primary_error_char = "~"; - *secondary_error_char = "^"; - break; - } - return 1; - } - case Subscript_kind: { - *left_anchor = expr->v.Subscript.value->end_col_offset; - *right_anchor = expr->v.Subscript.slice->end_col_offset + 1; - Py_ssize_t str_len = strlen(segment_str); - - // Move right_anchor and left_anchor forward to the first non-whitespace character that is not ']' and '[' - while (*left_anchor < str_len && (IS_WHITESPACE(segment_str[*left_anchor]) || segment_str[*left_anchor] != '[')) { - ++*left_anchor; - } - while (*right_anchor < str_len && (IS_WHITESPACE(segment_str[*right_anchor]) || segment_str[*right_anchor] != ']')) { - ++*right_anchor; - } - if (*right_anchor < str_len){ - *right_anchor += 1; - } - - // Set the error characters - *primary_error_char = "~"; - *secondary_error_char = "^"; - return 1; - } - default: - return 0; - } -} - -static int -extract_anchors_from_stmt(const char *segment_str, stmt_ty statement, Py_ssize_t *left_anchor, Py_ssize_t *right_anchor, - char** primary_error_char, char** secondary_error_char) -{ - switch (statement->kind) { - case Expr_kind: { - return extract_anchors_from_expr(segment_str, statement->v.Expr.value, left_anchor, right_anchor, - primary_error_char, secondary_error_char); - } - default: - return 0; - } -} - -static int -extract_anchors_from_line(PyObject *filename, PyObject *line, - Py_ssize_t start_offset, Py_ssize_t end_offset, - Py_ssize_t *left_anchor, Py_ssize_t *right_anchor, - char** primary_error_char, char** secondary_error_char) -{ - int res = -1; - PyArena *arena = NULL; - PyObject *segment = PyUnicode_Substring(line, start_offset, end_offset); - if (!segment) { - goto done; - } - - const char *segment_str = PyUnicode_AsUTF8(segment); - if (!segment_str) { - goto done; - } - - arena = _PyArena_New(); - if (!arena) { - goto done; - } - - PyCompilerFlags flags = _PyCompilerFlags_INIT; - - mod_ty module = _PyParser_ASTFromString(segment_str, filename, Py_file_input, - &flags, arena); - if (!module) { - goto done; - } - if (!_PyAST_Optimize(module, arena, _Py_GetConfig()->optimization_level, 0)) { - goto done; - } - - assert(module->kind == Module_kind); - if (asdl_seq_LEN(module->v.Module.body) == 1) { - stmt_ty statement = asdl_seq_GET(module->v.Module.body, 0); - res = extract_anchors_from_stmt(segment_str, statement, left_anchor, right_anchor, - primary_error_char, secondary_error_char); - } else { - res = 0; - } - -done: - if (res > 0) { - // Normalize the AST offsets to byte offsets and adjust them with the - // start of the actual line (instead of the source code segment). - assert(segment != NULL); - assert(*left_anchor >= 0); - assert(*right_anchor >= 0); - *left_anchor = _PyPegen_byte_offset_to_character_offset(segment, *left_anchor) + start_offset; - *right_anchor = _PyPegen_byte_offset_to_character_offset(segment, *right_anchor) + start_offset; - } - Py_XDECREF(segment); - if (arena) { - _PyArena_Free(arena); - } - return res; -} - #define _TRACEBACK_SOURCE_LINE_INDENT 4 static inline int @@ -742,30 +593,6 @@ ignore_source_errors(void) { return 0; } -static inline int -print_error_location_carets(PyObject *f, int offset, Py_ssize_t start_offset, Py_ssize_t end_offset, - Py_ssize_t right_start_offset, Py_ssize_t left_end_offset, - const char *primary, const char *secondary) { - int special_chars = (left_end_offset != -1 || right_start_offset != -1); - const char *str; - while (++offset <= end_offset) { - if (offset <= start_offset) { - str = " "; - } else if (special_chars && left_end_offset < offset && offset <= right_start_offset) { - str = secondary; - } else { - str = primary; - } - if (PyFile_WriteString(str, f) < 0) { - return -1; - } - } - if (PyFile_WriteString("\n", f) < 0) { - return -1; - } - return 0; -} - static int tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int lineno, PyFrameObject *frame, PyObject *name, int margin_indent, const char *margin) @@ -823,74 +650,18 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen goto done; } - // When displaying errors, we will use the following generic structure: - // - // ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE - // ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^~~~~~~~~~~~~~~~~~~~ - // | |-> left_end_offset | |-> end_offset - // |-> start_offset |-> right_start_offset - // - // In general we will only have (start_offset, end_offset) but we can gather more information - // by analyzing the AST of the text between *start_offset* and *end_offset*. If this succeeds - // we could get *left_end_offset* and *right_start_offset* and some selection of characters for - // the different ranges (primary_error_char and secondary_error_char). If we cannot obtain the - // AST information or we cannot identify special ranges within it, then left_end_offset and - // right_end_offset will be set to -1. - // - // To keep the column indicators pertinent, they are not shown when the primary character - // spans the whole line. - - // Convert the utf-8 byte offset to the actual character offset so we print the right number of carets. - assert(source_line); - Py_ssize_t start_offset = _PyPegen_byte_offset_to_character_offset(source_line, start_col_byte_offset); - if (start_offset < 0) { - err = ignore_source_errors() < 0; - goto done; - } - - Py_ssize_t end_offset = _PyPegen_byte_offset_to_character_offset(source_line, end_col_byte_offset); - if (end_offset < 0) { - err = ignore_source_errors() < 0; + // If this is a multi-line expression, then we will highlight until + // the last non-whitespace character. + const char *source_line_str = PyUnicode_AsUTF8(source_line); + if (!source_line_str) { goto done; } - Py_ssize_t left_end_offset = -1; - Py_ssize_t right_start_offset = -1; - - char *primary_error_char = "^"; - char *secondary_error_char = primary_error_char; - - if (start_line == end_line) { - int res = extract_anchors_from_line(filename, source_line, start_offset, end_offset, - &left_end_offset, &right_start_offset, - &primary_error_char, &secondary_error_char); - if (res < 0 && ignore_source_errors() < 0) { - goto done; - } - } - else { - // If this is a multi-line expression, then we will highlight until - // the last non-whitespace character. - const char *source_line_str = PyUnicode_AsUTF8(source_line); - if (!source_line_str) { - goto done; - } - - Py_ssize_t i = source_line_len; - while (--i >= 0) { - if (!IS_WHITESPACE(source_line_str[i])) { - break; - } + Py_ssize_t i = source_line_len; + while (--i >= 0) { + if (!IS_WHITESPACE(source_line_str[i])) { + break; } - - end_offset = i + 1; - } - - // Elide indicators if primary char spans the frame line - Py_ssize_t stripped_line_len = source_line_len - truncation - _TRACEBACK_SOURCE_LINE_INDENT; - bool has_secondary_ranges = (left_end_offset != -1 || right_start_offset != -1); - if (end_offset - start_offset == stripped_line_len && !has_secondary_ranges) { - goto done; } if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) { @@ -898,13 +669,6 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen goto done; } - if (print_error_location_carets(f, truncation, start_offset, end_offset, - right_start_offset, left_end_offset, - primary_error_char, secondary_error_char) < 0) { - err = -1; - goto done; - } - done: Py_XDECREF(source_line); return err; From 987988d6c6c5718a1825ebc9198b38d607f2f914 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 13:24:07 +0200 Subject: [PATCH 02/13] Remove old macros --- Python/pythonrun.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 029a07d511ac50..3aed57141afba2 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -687,13 +687,11 @@ struct exception_print_context PyObject *seen; // Prevent cycles in recursion bool need_close; // Need a closing bottom frame }; -#define EXC_MARGIN(ctx) ("") -#define EXC_INDENT(ctx) (0) static int write_indented_margin(struct exception_print_context *ctx, PyObject *f) { - return _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN(ctx), f); + return _Py_WriteIndentedMargin(0, "", f); } static int @@ -701,7 +699,7 @@ print_exception_invalid_type(struct exception_print_context *ctx, PyObject *value) { PyObject *f = ctx->file; - if (_Py_WriteIndent(EXC_INDENT(ctx), f) < 0) { + if (_Py_WriteIndent(0, f) < 0) { return -1; } const char *const msg = "TypeError: print_exception(): Exception expected " @@ -727,9 +725,9 @@ print_exception_traceback(struct exception_print_context *ctx, PyObject *value) PyObject *tb = PyException_GetTraceback(value); if (tb && tb != Py_None) { const char *header = EXCEPTION_TB_HEADER; - const char *header_margin = EXC_MARGIN(ctx); + const char *header_margin = ""; err = _PyTraceBack_Print_Indented( - tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), header_margin, header, f); + tb, 0, "", header_margin, header, f); } Py_XDECREF(tb); return err; From 2292ab38c79910594d33f24fe4163ceb0fbc4fe9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 14:15:30 +0200 Subject: [PATCH 03/13] Add tests for the fallback option --- Lib/test/test_traceback.py | 49 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index aa8405bd25d120..7e52df7dbf4098 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -963,7 +963,9 @@ class CPythonTracebackLegacyErrorCaretTests( Same set of tests as above but with Python's legacy internal traceback printing. """ -class TracebackFormatTests(unittest.TestCase): + +class TracebackFormatMixing: + DEBUG_RANGES = True def some_exception(self): raise KeyError('blah') @@ -1137,6 +1139,8 @@ def g(count=10): ) expected = (tb_line + result_g).splitlines() actual = stderr_g.getvalue().splitlines() + if not self.DEBUG_RANGES: + expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Check 2 different repetitive sections @@ -1173,6 +1177,8 @@ def h(count=10): ) expected = (result_h + result_g).splitlines() actual = stderr_h.getvalue().splitlines() + if not self.DEBUG_RANGES: + expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Check the boundary conditions. First, test just below the cutoff. @@ -1199,11 +1205,13 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+81}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' ) expected = (tb_line + result_g).splitlines() actual = stderr_g.getvalue().splitlines() + if not self.DEBUG_RANGES: + expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Second, test just above the cutoff. @@ -1231,24 +1239,24 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+108}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+114}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' ) expected = (tb_line + result_g).splitlines() actual = stderr_g.getvalue().splitlines() + if not self.DEBUG_RANGES: + expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) @requires_debug_ranges() - def test_recursive_traceback_python(self): - self._check_recursive_traceback_display(traceback.print_exc) - - @cpython_only - @requires_debug_ranges() - def test_recursive_traceback_cpython_internal(self): - from _testcapi import exception_print - def render_exc(): - exception_print(sys.exception()) - self._check_recursive_traceback_display(render_exc) + def test_recursive_traceback(self): + if self.DEBUG_RANGES: + self._check_recursive_traceback_display(traceback.print_exc) + else: + from _testcapi import exception_print + def render_exc(): + exception_print(sys.exception()) + self._check_recursive_traceback_display(render_exc) def test_format_stack(self): def fmt(): @@ -1345,6 +1353,21 @@ def test_print_exception_bad_type_python(self): boundaries = re.compile( '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) +class TestTracebackFormat(unittest.TestCase, TracebackFormatMixing): + pass + +@cpython_only +class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixing): + DEBUG_RANGES = False + def setUp(self) -> None: + self.original_hook = traceback._print_exception_bltin + traceback._print_exception_bltin = lambda *args: 1/0 + return super().setUp() + + def tearDown(self) -> None: + traceback._print_exception_bltin = self.original_hook + return super().tearDown() + class BaseExceptionReportingTests: def get_exception(self, exception_or_callable): From 2ab21688ce50298ff143f74d90fed63b2311d690 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 14:30:06 +0200 Subject: [PATCH 04/13] fixup! Add tests for the fallback option --- Python/pythonrun.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 3aed57141afba2..4f5c2cdd64a527 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -685,7 +685,6 @@ struct exception_print_context { PyObject *file; PyObject *seen; // Prevent cycles in recursion - bool need_close; // Need a closing bottom frame }; static int @@ -926,9 +925,7 @@ print_chained(struct exception_print_context* ctx, PyObject *value, if (_Py_EnterRecursiveCall(" in print_chained")) { return -1; } - bool need_close = ctx->need_close; int res = print_exception_recursive(ctx, value); - ctx->need_close = need_close; _Py_LeaveRecursiveCall(); if (res < 0) { return -1; @@ -1099,7 +1096,6 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) struct exception_print_context ctx; ctx.file = file; - ctx.need_close = false; /* We choose to ignore seen being possibly NULL, and report at least the main exception (it could be a MemoryError). From e15171777827ec7884a270c8877b832341b75b03 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 16:47:23 +0200 Subject: [PATCH 05/13] fixup! fixup! Add tests for the fallback option --- Lib/test/test_traceback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 7e52df7dbf4098..a2adc3ed8ef5eb 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1360,10 +1360,10 @@ class TestTracebackFormat(unittest.TestCase, TracebackFormatMixing): class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixing): DEBUG_RANGES = False def setUp(self) -> None: - self.original_hook = traceback._print_exception_bltin + self.original_hook = traceback._print_exception_bltin traceback._print_exception_bltin = lambda *args: 1/0 return super().setUp() - + def tearDown(self) -> None: traceback._print_exception_bltin = self.original_hook return super().tearDown() From 7d57970b6e3a7aff4404983c98b14963bdad4b34 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 16:56:57 +0200 Subject: [PATCH 06/13] fixup! fixup! fixup! Add tests for the fallback option --- .../2023-10-11-16-56-54.gh-issue-110721.afcSsH.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-11-16-56-54.gh-issue-110721.afcSsH.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-11-16-56-54.gh-issue-110721.afcSsH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-11-16-56-54.gh-issue-110721.afcSsH.rst new file mode 100644 index 00000000000000..299ac5ef308619 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-11-16-56-54.gh-issue-110721.afcSsH.rst @@ -0,0 +1,2 @@ +Use the :mod:`traceback` implementation for the default +:c:func:`PyErr_Display` functionality. Patch by Pablo Galindo From c812a3c59c0f6cbc75dc887cb998539df1dd7d99 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 17:04:08 +0200 Subject: [PATCH 07/13] fixup! fixup! fixup! fixup! Add tests for the fallback option --- Lib/test/test_traceback.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index a2adc3ed8ef5eb..dbcc94e029003d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -964,7 +964,7 @@ class CPythonTracebackLegacyErrorCaretTests( """ -class TracebackFormatMixing: +class TracebackFormatMixin: DEBUG_RANGES = True def some_exception(self): @@ -1353,11 +1353,11 @@ def test_print_exception_bad_type_python(self): boundaries = re.compile( '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) -class TestTracebackFormat(unittest.TestCase, TracebackFormatMixing): +class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin): pass @cpython_only -class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixing): +class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin): DEBUG_RANGES = False def setUp(self) -> None: self.original_hook = traceback._print_exception_bltin From d55b7e766175e69ff137fd169f4e04151a9f2047 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 17:12:01 +0200 Subject: [PATCH 08/13] fixup! fixup! fixup! fixup! fixup! Add tests for the fallback option --- Lib/traceback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 2adce608442268..12fcdad7dd4cb8 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -415,7 +415,7 @@ def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, # Only lineno is required, the remaining fields can be None if the # information is not available. builtin_limit = limit is BUILTIN_EXCEPTION_LIMIT - if limit is None or limit is BUILTIN_EXCEPTION_LIMIT: + if limit is None or builtin_limit: limit = getattr(sys, 'tracebacklimit', None) if limit is not None and limit < 0: limit = 0 From a65d281802dcb3fa93665c36b3dcb6a903bda808 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 17:27:02 +0200 Subject: [PATCH 09/13] fixup! fixup! fixup! fixup! fixup! fixup! Add tests for the fallback option --- Python/pythonrun.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 4f5c2cdd64a527..ec6ce7954c1050 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1092,6 +1092,9 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) return; } fallback: +#ifdef Py_DEBUG + _PyErr_WriteUnraisableMsg("in the internal traceback machinery", NULL); +#endif PyErr_Clear(); struct exception_print_context ctx; From 4d438ecd915069451b54715372ddcf8b035a7988 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 12 Oct 2023 09:31:49 +0200 Subject: [PATCH 10/13] fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add tests for the fallback option --- Lib/test/test_sys.py | 3 ++- Lib/test/test_traceback.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ae241d7a502749..42ee6a46679369 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -175,7 +175,8 @@ def test_excepthook_bytes_filename(self): def test_excepthook(self): with test.support.captured_output("stderr") as stderr: - sys.excepthook(1, '1', 1) + with test.support.catch_unraisable_exception(): + sys.excepthook(1, '1', 1) self.assertTrue("TypeError: print_exception(): Exception expected for " \ "value, str found" in stderr.getvalue()) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index dbcc94e029003d..09106a0242c08c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1360,12 +1360,15 @@ class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin): class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin): DEBUG_RANGES = False def setUp(self) -> None: + self.original_unraisable_hook = sys.unraisablehook + sys.unraisablehook = lambda *args: None self.original_hook = traceback._print_exception_bltin traceback._print_exception_bltin = lambda *args: 1/0 return super().setUp() def tearDown(self) -> None: traceback._print_exception_bltin = self.original_hook + sys.unraisablehook = self.original_unraisable_hook return super().tearDown() class BaseExceptionReportingTests: From dab41fb22e2896a58f702ab065387e96f842dc76 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 12 Oct 2023 10:47:54 +0200 Subject: [PATCH 11/13] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add tests for the fallback option --- Include/internal/pycore_traceback.h | 5 +-- Python/pythonrun.c | 45 ++++++--------------- Python/traceback.c | 61 ++++++----------------------- 3 files changed, 24 insertions(+), 87 deletions(-) diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 01b3f5b6b26d2e..4a23aeaca91ef1 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -95,9 +95,8 @@ extern PyObject* _PyTraceBack_FromFrame( /* Write the traceback tb to file f. Prefix each line with indent spaces followed by the margin (if it is not NULL). */ -extern int _PyTraceBack_Print_Indented( - PyObject *tb, int indent, const char* margin, - const char *header_margin, const char *header, PyObject *f); +extern int _PyTraceBack_Print( + PyObject *tb, const char *header_margin, const char *header, PyObject *f); extern int _Py_WriteIndentedMargin(int, const char*, PyObject *); extern int _Py_WriteIndent(int, PyObject *); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index ec6ce7954c1050..456c52b5f2c09b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -23,7 +23,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pythonrun.h" // export _PyRun_InteractiveLoopObject() #include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_traceback.h" // _PyTraceBack_Print_Indented() +#include "pycore_traceback.h" // _PyTraceBack_Print() #include "errcode.h" // E_EOF #include "marshal.h" // PyMarshal_ReadLongFromFile() @@ -687,20 +687,11 @@ struct exception_print_context PyObject *seen; // Prevent cycles in recursion }; -static int -write_indented_margin(struct exception_print_context *ctx, PyObject *f) -{ - return _Py_WriteIndentedMargin(0, "", f); -} - static int print_exception_invalid_type(struct exception_print_context *ctx, PyObject *value) { PyObject *f = ctx->file; - if (_Py_WriteIndent(0, f) < 0) { - return -1; - } const char *const msg = "TypeError: print_exception(): Exception expected " "for value, "; if (PyFile_WriteString(msg, f) < 0) { @@ -725,8 +716,7 @@ print_exception_traceback(struct exception_print_context *ctx, PyObject *value) if (tb && tb != Py_None) { const char *header = EXCEPTION_TB_HEADER; const char *header_margin = ""; - err = _PyTraceBack_Print_Indented( - tb, 0, "", header_margin, header, f); + err = _PyTraceBack_Print(tb, header_margin, header, f); } Py_XDECREF(tb); return err; @@ -769,9 +759,6 @@ print_exception_file_and_line(struct exception_print_context *ctx, if (line == NULL) { goto error; } - if (write_indented_margin(ctx, f) < 0) { - goto error; - } if (PyFile_WriteObject(line, f, Py_PRINT_RAW) < 0) { goto error; } @@ -794,9 +781,6 @@ print_exception_message(struct exception_print_context *ctx, PyObject *type, assert(PyExceptionClass_Check(type)); - if (write_indented_margin(ctx, f) < 0) { - return -1; - } PyObject *modulename = PyObject_GetAttr(type, &_Py_ID(__module__)); if (modulename == NULL || !PyUnicode_Check(modulename)) { Py_XDECREF(modulename); @@ -931,21 +915,12 @@ print_chained(struct exception_print_context* ctx, PyObject *value, return -1; } - if (write_indented_margin(ctx, f) < 0) { - return -1; - } if (PyFile_WriteString("\n", f) < 0) { return -1; } - if (write_indented_margin(ctx, f) < 0) { - return -1; - } if (PyFile_WriteString(message, f) < 0) { return -1; } - if (write_indented_margin(ctx, f) < 0) { - return -1; - } if (PyFile_WriteString("\n", f) < 0) { return -1; } @@ -1063,35 +1038,37 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } } + int unhandled_keyboard_interrupt = _PyRuntime.signals.unhandled_keyboard_interrupt; + if (!value) { goto fallback; } - // Try first with the traceback. - int unhandled_keyboard_interrupt = _PyRuntime.signals.unhandled_keyboard_interrupt; + // Try first with the stdlib traceback module PyObject *traceback_module = PyImport_ImportModule("traceback"); if (traceback_module == NULL) { goto fallback; } - PyObject *print_tb_func = PyObject_GetAttrString(traceback_module, "_print_exception_bltin"); + PyObject *print_exception_fn = PyObject_GetAttrString(traceback_module, "_print_exception_bltin"); - if (print_tb_func == NULL || !PyCallable_Check(print_tb_func)) { + if (print_exception_fn == NULL || !PyCallable_Check(print_exception_fn)) { Py_DECREF(traceback_module); goto fallback; } - PyObject* result = PyObject_CallOneArg(print_tb_func, value); + PyObject* result = PyObject_CallOneArg(print_exception_fn, value); Py_DECREF(traceback_module); - Py_XDECREF(print_tb_func); - _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt; + Py_XDECREF(print_exception_fn); if (result) { Py_DECREF(result); + _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt; return; } fallback: + _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt; #ifdef Py_DEBUG _PyErr_WriteUnraisableMsg("in the internal traceback machinery", NULL); #endif diff --git a/Python/traceback.c b/Python/traceback.c index 0f995a0df700a5..42fe532b439df7 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -397,27 +397,9 @@ _Py_WriteIndent(int indent, PyObject *f) return 0; } -/* Writes indent spaces, followed by the margin if it is not `\0`. - Returns 0 on success and non-zero on failure. - */ -int -_Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f) -{ - if (_Py_WriteIndent(indent, f) < 0) { - return -1; - } - if (margin) { - if (PyFile_WriteString(margin, f) < 0) { - return -1; - } - } - return 0; -} - static int -display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int indent, - int margin_indent, const char *margin, - int *truncation, PyObject **line) +display_source_line(PyObject *f, PyObject *filename, int lineno, int indent, + int *truncation, PyObject **line) { int fd; int i; @@ -545,10 +527,6 @@ display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int *truncation = i - indent; } - if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) { - goto error; - } - /* Write some spaces before the line */ if (_Py_WriteIndent(indent, f) < 0) { goto error; @@ -574,8 +552,7 @@ int _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, int *truncation, PyObject **line) { - return display_source_line_with_margin(f, filename, lineno, indent, 0, - NULL, truncation, line); + return display_source_line(f, filename, lineno, indent, truncation, line); } @@ -595,16 +572,12 @@ ignore_source_errors(void) { static int tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int lineno, - PyFrameObject *frame, PyObject *name, int margin_indent, const char *margin) + PyFrameObject *frame, PyObject *name) { if (filename == NULL || name == NULL) { return -1; } - if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) { - return -1; - } - PyObject *line = PyUnicode_FromFormat(" File \"%U\", line %d, in %U\n", filename, lineno, name); if (line == NULL) { @@ -621,9 +594,9 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen int truncation = _TRACEBACK_SOURCE_LINE_INDENT; PyObject* source_line = NULL; - int rc = display_source_line_with_margin( + int rc = display_source_line( f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT, - margin_indent, margin, &truncation, &source_line); + &truncation, &source_line); if (rc != 0 || !source_line) { /* ignore errors since we can't report them, can we? */ err = ignore_source_errors(); @@ -664,11 +637,6 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen } } - if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) { - err = -1; - goto done; - } - done: Py_XDECREF(source_line); return err; @@ -694,8 +662,7 @@ tb_print_line_repeated(PyObject *f, long cnt) } static int -tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, - int indent, const char *margin) +tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) { PyCodeObject *code = NULL; Py_ssize_t depth = 0; @@ -731,7 +698,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, cnt++; if (cnt <= TB_RECURSIVE_CUTOFF) { if (tb_displayline(tb, f, code->co_filename, tb->tb_lineno, - tb->tb_frame, code->co_name, indent, margin) < 0) { + tb->tb_frame, code->co_name) < 0) { goto error; } @@ -756,8 +723,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, #define PyTraceBack_LIMIT 1000 int -_PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, - const char *header_margin, const char *header, PyObject *f) +_PyTraceBack_Print(PyObject *v, const char *header_margin, const char *header, PyObject *f) { PyObject *limitv; long limit = PyTraceBack_LIMIT; @@ -780,15 +746,12 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, return 0; } } - if (_Py_WriteIndentedMargin(indent, header_margin, f) < 0) { - return -1; - } if (PyFile_WriteString(header, f) < 0) { return -1; } - if (tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin) < 0) { + if (tb_printinternal((PyTracebackObject *)v, f, limit) < 0) { return -1; } @@ -798,12 +761,10 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, int PyTraceBack_Print(PyObject *v, PyObject *f) { - int indent = 0; - const char *margin = NULL; const char *header_margin = NULL; const char *header = EXCEPTION_TB_HEADER; - return _PyTraceBack_Print_Indented(v, indent, margin, header_margin, header, f); + return _PyTraceBack_Print(v, header_margin, header, f); } /* Format an integer in range [0; 0xffffffff] to decimal and write it From a6eb00e2ae8dd0621926cc20ae38e46c4143fc38 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 12 Oct 2023 12:10:42 +0200 Subject: [PATCH 12/13] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add tests for the fallback option --- Include/internal/pycore_traceback.h | 2 +- Python/pythonrun.c | 16 +++++++++++----- Python/traceback.c | 6 ++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 4a23aeaca91ef1..10922bff98bd4b 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -96,7 +96,7 @@ extern PyObject* _PyTraceBack_FromFrame( /* Write the traceback tb to file f. Prefix each line with indent spaces followed by the margin (if it is not NULL). */ extern int _PyTraceBack_Print( - PyObject *tb, const char *header_margin, const char *header, PyObject *f); + PyObject *tb, const char *header, PyObject *f); extern int _Py_WriteIndentedMargin(int, const char*, PyObject *); extern int _Py_WriteIndent(int, PyObject *); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 456c52b5f2c09b..2e2747158c9ad4 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -715,8 +715,7 @@ print_exception_traceback(struct exception_print_context *ctx, PyObject *value) PyObject *tb = PyException_GetTraceback(value); if (tb && tb != Py_None) { const char *header = EXCEPTION_TB_HEADER; - const char *header_margin = ""; - err = _PyTraceBack_Print(tb, header_margin, header, f); + err = _PyTraceBack_Print(tb, header, f); } Py_XDECREF(tb); return err; @@ -779,6 +778,12 @@ print_exception_message(struct exception_print_context *ctx, PyObject *type, { PyObject *f = ctx->file; + if (PyErr_GivenExceptionMatches(value, PyExc_MemoryError)) { + // The Python APIs in this function require allocating memory + // for various objects. If we're out of memory, we can't do that, + return -1; + } + assert(PyExceptionClass_Check(type)); PyObject *modulename = PyObject_GetAttr(type, &_Py_ID(__module__)); @@ -1040,7 +1045,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) int unhandled_keyboard_interrupt = _PyRuntime.signals.unhandled_keyboard_interrupt; - if (!value) { + if (!value || PyErr_GivenExceptionMatches(value, PyExc_MemoryError)) { goto fallback; } @@ -1070,10 +1075,11 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) fallback: _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt; #ifdef Py_DEBUG - _PyErr_WriteUnraisableMsg("in the internal traceback machinery", NULL); + if (PyErr_Occurred()) { + _PyErr_WriteUnraisableMsg("in the internal traceback machinery", NULL); + } #endif PyErr_Clear(); - struct exception_print_context ctx; ctx.file = file; diff --git a/Python/traceback.c b/Python/traceback.c index 42fe532b439df7..fc75d43c2f78df 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -723,7 +723,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) #define PyTraceBack_LIMIT 1000 int -_PyTraceBack_Print(PyObject *v, const char *header_margin, const char *header, PyObject *f) +_PyTraceBack_Print(PyObject *v, const char *header, PyObject *f) { PyObject *limitv; long limit = PyTraceBack_LIMIT; @@ -761,10 +761,8 @@ _PyTraceBack_Print(PyObject *v, const char *header_margin, const char *header, P int PyTraceBack_Print(PyObject *v, PyObject *f) { - const char *header_margin = NULL; const char *header = EXCEPTION_TB_HEADER; - - return _PyTraceBack_Print(v, header_margin, header, f); + return _PyTraceBack_Print(v, header, f); } /* Format an integer in range [0; 0xffffffff] to decimal and write it From d74b83757cf5b7dd43d5ab9e79092937a20c6d28 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 12 Oct 2023 16:21:22 +0200 Subject: [PATCH 13/13] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add tests for the fallback option --- Lib/test/test_traceback.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 09106a0242c08c..9bb1786c5472f5 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1329,7 +1329,8 @@ def test_exception_group_deep_recursion_traceback(self): def test_print_exception_bad_type_capi(self): from _testcapi import exception_print with captured_output("stderr") as stderr: - exception_print(42) + with support.catch_unraisable_exception(): + exception_print(42) self.assertEqual( stderr.getvalue(), ('TypeError: print_exception(): '