From fc9b354f3078527656bb9267d68a36a54acf3da0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 2 Nov 2023 01:12:34 +0800 Subject: [PATCH 1/3] Split up tier 2 opcode into opcode and target --- Include/internal/pycore_uops.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index d8a7d978f1304e..a681232ef80f3e 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -13,7 +13,8 @@ extern "C" { #define _Py_UOP_MAX_TRACE_LENGTH 128 typedef struct { - uint32_t opcode; + uint16_t opcode; + uint16_t target; // Guard failure jump target. uint32_t oparg; uint64_t operand; // A cache entry } _PyUOpInstruction; From 326cb74271c48b576cfac391b7b5dda075a19e57 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 6 Nov 2023 08:21:36 +0800 Subject: [PATCH 2/3] Add target --- Python/executor.c | 15 +++++++++++---- Python/optimizer.c | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Python/executor.c b/Python/executor.c index 2884565689171c..7f148a095d5bde 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -23,10 +23,15 @@ #undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - UOP_STAT_INC(INSTNAME, miss); \ - goto deoptimize; \ +#define DEOPT_IF(COND, INSTNAME) \ + if ((COND)) { \ + if (target) { \ + pc = target; \ + } \ + else { \ + UOP_STAT_INC(INSTNAME, miss); \ + goto deoptimize; \ + } \ } #ifdef Py_STATS @@ -76,6 +81,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject int pc = 0; int opcode; int oparg; + int target; uint64_t operand; #ifdef Py_STATS uint64_t trace_uop_execution_counter = 0; @@ -83,6 +89,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject for (;;) { opcode = self->trace[pc].opcode; + target = self->trace[pc].target; oparg = self->trace[pc].oparg; operand = self->trace[pc].operand; DPRINTF(3, diff --git a/Python/optimizer.c b/Python/optimizer.c index c239b03053a896..b571fca5d6743c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -459,6 +459,7 @@ translate_bytecode_to_trace( trace[trace_length].opcode = (OPCODE); \ trace[trace_length].oparg = (OPARG); \ trace[trace_length].operand = (OPERAND); \ + trace[trace_length].target = 0 \ trace_length++; #define INSTR_IP(INSTR, CODE) \ From b0933e8b30f24e3040371fc7cc6466cab2682a8f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 6 Nov 2023 08:52:19 +0800 Subject: [PATCH 3/3] tier 2 deopts --- Python/ceval.c | 7 + Python/executor.c | 156 ----------------- Python/executor_cases.c.h | 240 +++++++++++++------------- Python/optimizer.c | 4 +- Tools/cases_generator/instructions.py | 5 +- 5 files changed, 133 insertions(+), 279 deletions(-) delete mode 100644 Python/executor.c diff --git a/Python/ceval.c b/Python/ceval.c index 670f312f126201..cac770f32500bc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -960,6 +960,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if ((COND)) { \ goto deoptimize;\ } +#undef TIER2_DEOPT_IF + +#define TIER2_DEOPT_IF(COND, INSTNAME) \ + if ((COND)) { \ + next_uop = current_executor->trace + (next_uop-1)->target; \ + break; \ + } #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 diff --git a/Python/executor.c b/Python/executor.c deleted file mode 100644 index 7f148a095d5bde..00000000000000 --- a/Python/executor.c +++ /dev/null @@ -1,156 +0,0 @@ -#include "Python.h" - -#include "opcode.h" - -#include "pycore_bitutils.h" -#include "pycore_call.h" -#include "pycore_ceval.h" -#include "pycore_dict.h" -#include "pycore_emscripten_signal.h" -#include "pycore_intrinsics.h" -#include "pycore_long.h" -#include "pycore_object.h" -#include "pycore_opcode_metadata.h" -#include "pycore_opcode_utils.h" -#include "pycore_pyerrors.h" -#include "pycore_range.h" -#include "pycore_setobject.h" // _PySet_Update() -#include "pycore_sliceobject.h" -#include "pycore_uops.h" - -#define TIER_TWO 2 -#include "ceval_macros.h" - - -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - if (target) { \ - pc = target; \ - } \ - else { \ - UOP_STAT_INC(INSTNAME, miss); \ - goto deoptimize; \ - } \ - } - -#ifdef Py_STATS -// Disable these macros that apply to Tier 1 stats when we are in Tier 2 -#undef STAT_INC -#define STAT_INC(opname, name) ((void)0) -#undef STAT_DEC -#define STAT_DEC(opname, name) ((void)0) -#undef CALL_STAT_INC -#define CALL_STAT_INC(name) ((void)0) -#endif - -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION 0 - - -_PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) -{ -#ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that - } - #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { printf(__VA_ARGS__); } -#else - #define DPRINTF(level, ...) -#endif - - DPRINTF(3, - "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n", - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), - _PyFrame_GetCode(frame)->co_firstlineno, - 2 * (long)(frame->instr_ptr - - (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); - - PyThreadState *tstate = _PyThreadState_GET(); - _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; - - CHECK_EVAL_BREAKER(); - - OPT_STAT_INC(traces_executed); - _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - int pc = 0; - int opcode; - int oparg; - int target; - uint64_t operand; -#ifdef Py_STATS - uint64_t trace_uop_execution_counter = 0; -#endif - - for (;;) { - opcode = self->trace[pc].opcode; - target = self->trace[pc].target; - oparg = self->trace[pc].oparg; - operand = self->trace[pc].operand; - DPRINTF(3, - "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", - pc, - opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], - oparg, - operand, - (int)(stack_pointer - _PyFrame_Stackbase(frame))); - pc++; - OPT_STAT_INC(uops_executed); - UOP_STAT_INC(opcode, execution_count); -#ifdef Py_STATS - trace_uop_execution_counter++; -#endif - - switch (opcode) { - -#include "executor_cases.c.h" - - default: - { - fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand); - Py_FatalError("Unknown uop"); - } - - } - } - -unbound_local_error: - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - // On ERROR_IF we return NULL as the frame. - // The caller recovers the frame from tstate->current_frame. - DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->return_offset = 0; // Don't leave this random - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return NULL; - -deoptimize: - // On DEOPT_IF we just repeat the last instruction. - // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->return_offset = 0; // Dispatch to frame->instr_ptr - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return frame; -} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index eb56c34b432783..d185f88e6fc44c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,13 +14,13 @@ case RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + TIER2_DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + TIER2_DEOPT_IF(eval_breaker != version, RESUME); break; } @@ -134,7 +134,7 @@ case TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + TIER2_DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); break; } @@ -143,7 +143,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + TIER2_DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -161,7 +161,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + TIER2_DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -174,7 +174,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + TIER2_DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -185,7 +185,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + TIER2_DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -207,7 +207,7 @@ uint32_t version = (uint32_t)next_uop[-1].operand; // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + TIER2_DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; @@ -231,8 +231,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); - DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); + TIER2_DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); + TIER2_DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); break; } @@ -289,8 +289,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); - DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); + TIER2_DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); + TIER2_DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); break; } @@ -347,8 +347,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); - DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); + TIER2_DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); + TIER2_DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); break; } @@ -439,13 +439,13 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + TIER2_DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + TIER2_DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -463,14 +463,14 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + TIER2_DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + TIER2_DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + TIER2_DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -486,13 +486,13 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + TIER2_DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + TIER2_DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -510,7 +510,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + TIER2_DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); if (res == NULL) { @@ -575,14 +575,14 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + TIER2_DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + TIER2_DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + TIER2_DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + TIER2_DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -602,7 +602,7 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + TIER2_DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -906,8 +906,8 @@ PyObject **values; seq = stack_pointer[-1]; values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + TIER2_DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + TIER2_DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -923,8 +923,8 @@ PyObject **values; seq = stack_pointer[-1]; values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + TIER2_DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + TIER2_DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -941,8 +941,8 @@ PyObject **values; seq = stack_pointer[-1]; values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + TIER2_DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + TIER2_DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1148,8 +1148,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)next_uop[-1].operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); + TIER2_DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); + TIER2_DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1157,8 +1157,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)next_uop[-1].operand; PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); + TIER2_DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); + TIER2_DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1170,7 +1170,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); + TIER2_DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1188,7 +1188,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); + TIER2_DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1539,8 +1539,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + TIER2_DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + TIER2_DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1563,8 +1563,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + TIER2_DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + TIER2_DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1636,7 +1636,7 @@ uint32_t type_version = (uint32_t)next_uop[-1].operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); + TIER2_DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); break; } @@ -1646,7 +1646,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); + TIER2_DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); break; } @@ -1658,7 +1658,7 @@ uint16_t index = (uint16_t)next_uop[-1].operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); + TIER2_DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1673,10 +1673,10 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)next_uop[-1].operand; - DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); + TIER2_DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); + TIER2_DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); break; } @@ -1691,7 +1691,7 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); + TIER2_DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1707,9 +1707,9 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); + TIER2_DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); + TIER2_DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1722,19 +1722,19 @@ uint16_t hint = (uint16_t)next_uop[-1].operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); + TIER2_DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + TIER2_DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + TIER2_DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); attr = ep->me_value; } - DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); + TIER2_DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1753,7 +1753,7 @@ uint16_t index = (uint16_t)next_uop[-1].operand; char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); + TIER2_DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1768,9 +1768,9 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)next_uop[-1].operand; - DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); + TIER2_DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); + TIER2_DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); break; } @@ -1796,7 +1796,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); + TIER2_DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); break; } @@ -1866,8 +1866,8 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + TIER2_DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + TIER2_DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -1888,10 +1888,10 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + TIER2_DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + TIER2_DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + TIER2_DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + TIER2_DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -1914,8 +1914,8 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + TIER2_DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + TIER2_DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2152,7 +2152,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); + TIER2_DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); break; } @@ -2197,7 +2197,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); + TIER2_DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); break; } @@ -2243,7 +2243,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); + TIER2_DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); break; } @@ -2378,7 +2378,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); + TIER2_DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); break; } @@ -2388,7 +2388,7 @@ uint32_t keys_version = (uint32_t)next_uop[-1].operand; PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); + TIER2_DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); break; } @@ -2466,7 +2466,7 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); + TIER2_DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); break; } @@ -2493,8 +2493,8 @@ PyObject *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + TIER2_DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + TIER2_DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); break; } @@ -2515,7 +2515,7 @@ } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); + TIER2_DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); break; } @@ -2525,11 +2525,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)next_uop[-1].operand; - DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); + TIER2_DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); + TIER2_DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); + TIER2_DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); break; } @@ -2538,8 +2538,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); - DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); + TIER2_DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); + TIER2_DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); break; } @@ -2600,9 +2600,9 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); + TIER2_DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + TIER2_DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); @@ -2622,8 +2622,8 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + TIER2_DEOPT_IF(null != NULL, CALL); + TIER2_DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PyObject_Str(arg); @@ -2646,8 +2646,8 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + TIER2_DEOPT_IF(null != NULL, CALL); + TIER2_DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); @@ -2688,9 +2688,9 @@ args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); + TIER2_DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + TIER2_DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -2720,9 +2720,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + TIER2_DEOPT_IF(total_args != 1, CALL); + TIER2_DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + TIER2_DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -2759,8 +2759,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + TIER2_DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + TIER2_DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -2802,8 +2802,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + TIER2_DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + TIER2_DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = @@ -2839,9 +2839,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + TIER2_DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + TIER2_DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -2874,9 +2874,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + TIER2_DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + TIER2_DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -2911,13 +2911,13 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + TIER2_DEOPT_IF(total_args != 2, CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + TIER2_DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -2953,12 +2953,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + TIER2_DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = @@ -2993,13 +2993,13 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + TIER2_DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + TIER2_DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3035,11 +3035,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + TIER2_DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + TIER2_DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; diff --git a/Python/optimizer.c b/Python/optimizer.c index 8f78c4a3442fa7..048560219d402c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -457,9 +457,9 @@ translate_bytecode_to_trace( assert(reserved > 0); \ reserved--; \ trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].oparg = (OPARG); \ + trace[trace_length].oparg = (OPARG); \ trace[trace_length].operand = (OPERAND); \ - trace[trace_length].target = 0 \ + trace[trace_length].target = OPCODE_HAS_DEOPT(OPCODE) ? (max_length + 2) : 0; \ trace_length++; #define INSTR_IP(INSTR, CODE) \ diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index f3e4f76697853b..4c8fbe7437b034 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -209,7 +209,10 @@ def write_body( space, cond = m.groups() space = extra + space target = family.name if family else self.name - out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") + if tier == TIER_TWO: + out.write_raw(f"{space}TIER2_DEOPT_IF({cond}, {target});\n") + else: + out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") elif "DEOPT" in line: filename = context.owner.filename lineno = context.owner.tokens[context.begin].line