From 82d166bd42bcf51877d6db70b8ca58481dd59092 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 27 Oct 2023 11:28:15 +0100 Subject: [PATCH 1/6] Use instr_ptr instead of next_instr-1 in many places --- Include/internal/pycore_opcode_metadata.h | 1 + Python/abstract_interp_cases.c.h | 8 ++ Python/bytecodes.c | 103 +++++++++++---------- Python/executor_cases.c.h | 83 +++++++++++++++-- Python/generated_cases.c.h | 107 ++++++++++++---------- 5 files changed, 196 insertions(+), 106 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e2ed9bad7833a5..4976d72f5b7055 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1687,6 +1687,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, + [LOAD_SUPER_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 232c569a3ddb30..82b26f8cbea1a4 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -438,6 +438,14 @@ break; } + case LOAD_SUPER_ATTR: { + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + case LOAD_SUPER_ATTR_ATTR: { STACK_SHRINK(2); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2d7b5ba21ea09d..493be36f3b28b9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -144,7 +144,7 @@ dummy_func( if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); - next_instr--; + next_instr = frame->instr_ptr; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { @@ -172,18 +172,19 @@ dummy_func( if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr--; + next_instr = frame->instr_ptr; } else { + _Py_CODEUNIT *here = frame->instr_ptr; if (oparg < 2) { CHECK_EVAL_BREAKER(); } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); + tstate, oparg > 0, frame, frame->instr_ptr); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); - if (frame->instr_ptr != next_instr-1) { + if (frame->instr_ptr != here) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; DISPATCH(); @@ -261,11 +262,12 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -278,9 +280,10 @@ dummy_func( } inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -312,7 +315,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } @@ -532,7 +535,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -681,7 +684,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -754,7 +757,7 @@ dummy_func( case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, frame->instr_ptr); goto exception_unwind; } break; @@ -808,7 +811,7 @@ dummy_func( inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, frame->instr_ptr, retval); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -832,7 +835,7 @@ dummy_func( PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, frame->instr_ptr, retval); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -960,7 +963,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } @@ -993,7 +996,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, frame->instr_ptr); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -1027,13 +1030,14 @@ dummy_func( inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + _Py_CODEUNIT *here = frame->instr_ptr; frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); + frame, here, retval); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1077,6 +1081,7 @@ dummy_func( } inst(RERAISE, (values[oparg], exc -- values[oparg])) { + _Py_CODEUNIT *here = frame->instr_ptr; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1093,7 +1098,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, here); goto exception_unwind; } @@ -1105,7 +1110,7 @@ dummy_func( else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, frame->instr_ptr); goto exception_unwind; } } @@ -1120,7 +1125,7 @@ dummy_func( } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, frame->instr_ptr); goto exception_unwind; } } @@ -1186,7 +1191,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -1250,7 +1255,7 @@ dummy_func( _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1372,7 +1377,7 @@ dummy_func( _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1711,7 +1716,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } @@ -1723,7 +1728,7 @@ dummy_func( PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); + frame, frame->instr_ptr, global_super, arg); ERROR_IF(err, error); } @@ -1736,12 +1741,12 @@ dummy_func( if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); + frame, frame->instr_ptr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); + frame, frame->instr_ptr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -1822,7 +1827,7 @@ dummy_func( _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2144,7 +2149,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -2277,7 +2282,7 @@ dummy_func( inst(JUMP_BACKWARD, (--)) { CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION @@ -2328,19 +2333,21 @@ dummy_func( } inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) { + _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } inst(POP_JUMP_IF_TRUE, (unused/1, cond -- )) { + _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -2458,7 +2465,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -2472,7 +2479,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, frame->instr_ptr); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2920,7 +2927,7 @@ dummy_func( &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); + frame, frame->instr_ptr, function, arg); ERROR_IF(err, error); _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -2961,7 +2968,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } @@ -3012,12 +3019,12 @@ dummy_func( if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3615,12 +3622,12 @@ dummy_func( if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3663,18 +3670,18 @@ dummy_func( PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); + frame, frame->instr_ptr, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); + frame, frame->instr_ptr, func, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); + frame, frame->instr_ptr, func, arg); if (err < 0) { Py_CLEAR(result); } @@ -3820,7 +3827,7 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } @@ -3842,9 +3849,9 @@ dummy_func( inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); + tstate, frame, frame->instr_ptr); ERROR_IF(next_opcode < 0, error); - next_instr--; + next_instr = frame->instr_ptr; if (_PyOpcode_Caches[next_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3872,7 +3879,7 @@ dummy_func( int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } @@ -3884,7 +3891,7 @@ dummy_func( int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } @@ -3902,7 +3909,7 @@ dummy_func( offset = 0; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } @@ -3920,7 +3927,7 @@ dummy_func( offset = oparg; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | !nflag; + here[1].cache = (here[1].cache << 1) | !nflag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9f37cd75b47efb..0c52afaec981e5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -121,7 +121,7 @@ #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } @@ -382,7 +382,7 @@ #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -576,7 +576,7 @@ #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -902,7 +902,7 @@ #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -992,7 +992,7 @@ _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1136,7 +1136,7 @@ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1559,6 +1559,71 @@ break; } + case LOAD_SUPER_ATTR: { + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *null = NULL; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + int load_method = oparg & 1; + #if ENABLE_SPECIALIZATION + _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + next_instr = frame->instr_ptr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_SUPER_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + #endif /* ENABLE_SPECIALIZATION */ + + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, frame->instr_ptr, global_super, arg); + if (err) goto pop_3_error; + } + + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, frame->instr_ptr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, frame->instr_ptr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } + } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + break; + } + case LOAD_SUPER_ATTR_ATTR: { PyObject *self; PyObject *class; @@ -1627,7 +1692,7 @@ _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1889,7 +1954,7 @@ #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3211,7 +3276,7 @@ #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d5e0d849efe9d2..78a91f9c4d7400 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -20,7 +20,7 @@ if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; - next_instr--; + next_instr = frame->instr_ptr; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { @@ -50,18 +50,19 @@ if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr--; + next_instr = frame->instr_ptr; } else { + _Py_CODEUNIT *here = frame->instr_ptr; if (oparg < 2) { CHECK_EVAL_BREAKER(); } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); + tstate, oparg > 0, frame, frame->instr_ptr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; - if (frame->instr_ptr != next_instr-1) { + if (frame->instr_ptr != here) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; DISPATCH(); @@ -195,11 +196,12 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; + TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -226,9 +228,10 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; + TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -269,7 +272,7 @@ #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } @@ -612,7 +615,7 @@ #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -843,7 +846,7 @@ #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -962,7 +965,7 @@ case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, frame->instr_ptr); goto exception_unwind; } break; @@ -1017,7 +1020,7 @@ retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, frame->instr_ptr, retval); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -1071,7 +1074,7 @@ PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, frame->instr_ptr, retval); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -1218,7 +1221,7 @@ #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } @@ -1251,7 +1254,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, frame->instr_ptr); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -1294,13 +1297,14 @@ retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + _Py_CODEUNIT *here = frame->instr_ptr; frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); + frame, here, retval); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1354,6 +1358,7 @@ PyObject **values; exc = stack_pointer[-1]; values = stack_pointer - 1 - oparg; + _Py_CODEUNIT *here = frame->instr_ptr; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1370,7 +1375,7 @@ assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, here); goto exception_unwind; } @@ -1387,7 +1392,7 @@ else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, frame->instr_ptr); goto exception_unwind; } STACK_SHRINK(2); @@ -1414,7 +1419,7 @@ } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, frame->instr_ptr); goto exception_unwind; } STACK_SHRINK(1); @@ -1494,7 +1499,7 @@ #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -1590,7 +1595,7 @@ _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1737,7 +1742,7 @@ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2211,7 +2216,7 @@ #if ENABLE_SPECIALIZATION _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } @@ -2223,7 +2228,7 @@ PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); + frame, frame->instr_ptr, global_super, arg); if (err) goto pop_3_error; } @@ -2236,12 +2241,12 @@ if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); + frame, frame->instr_ptr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); + frame, frame->instr_ptr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -2335,7 +2340,7 @@ _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2752,7 +2757,7 @@ #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3011,10 +3016,11 @@ TARGET(POP_JUMP_IF_FALSE) { PyObject *cond; cond = stack_pointer[-1]; + _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); STACK_SHRINK(1); @@ -3025,10 +3031,11 @@ TARGET(POP_JUMP_IF_TRUE) { PyObject *cond; cond = stack_pointer[-1]; + _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); STACK_SHRINK(1); @@ -3054,10 +3061,11 @@ // POP_JUMP_IF_TRUE cond = b; { + _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -3084,10 +3092,11 @@ // POP_JUMP_IF_FALSE cond = b; { + _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -3236,7 +3245,7 @@ #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3250,7 +3259,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, frame->instr_ptr); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -3766,7 +3775,7 @@ &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); + frame, frame->instr_ptr, function, arg); if (err) goto error; _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3792,7 +3801,7 @@ #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } @@ -3843,12 +3852,12 @@ if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -4721,12 +4730,12 @@ if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, frame->instr_ptr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -4781,18 +4790,18 @@ PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); + frame, frame->instr_ptr, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); + frame, frame->instr_ptr, func, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); + frame, frame->instr_ptr, func, arg); if (err < 0) { Py_CLEAR(result); } @@ -5000,7 +5009,7 @@ #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + next_instr = frame->instr_ptr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } @@ -5033,9 +5042,9 @@ TARGET(INSTRUMENTED_INSTRUCTION) { int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); + tstate, frame, frame->instr_ptr); if (next_opcode < 0) goto error; - next_instr--; + next_instr = frame->instr_ptr; if (_PyOpcode_Caches[next_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -5065,7 +5074,7 @@ int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); next_instr += 1; @@ -5079,7 +5088,7 @@ int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); next_instr += 1; @@ -5099,7 +5108,7 @@ offset = 0; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); next_instr += 1; @@ -5119,7 +5128,7 @@ offset = oparg; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | !nflag; + here[1].cache = (here[1].cache << 1) | !nflag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); next_instr += 1; From e86ea39912c8387510b48098b6a988a8912f857b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 27 Oct 2023 12:50:58 +0100 Subject: [PATCH 2/6] Move next_instr increment to start of instruction. --- Include/internal/pycore_opcode_metadata.h | 12 +- Lib/test/test_generated_cases.py | 90 +- Lib/test/test_monitoring.py | 18 +- Python/abstract_interp_cases.c.h | 8 - Python/bytecodes.c | 168 ++-- Python/ceval.c | 2 +- Python/ceval_macros.h | 12 +- Python/executor_cases.c.h | 112 +-- Python/generated_cases.c.h | 963 +++++++++++++++++----- Tools/cases_generator/analysis.py | 1 - Tools/cases_generator/instructions.py | 5 +- Tools/cases_generator/stacking.py | 13 +- 12 files changed, 935 insertions(+), 469 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4976d72f5b7055..27948749d58eb7 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1356,8 +1356,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, @@ -1434,7 +1434,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_ZERO_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, @@ -1495,7 +1495,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_IS_ITER_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, 0 }, @@ -1533,7 +1533,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1687,7 +1687,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, - [LOAD_SUPER_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, @@ -2048,6 +2047,7 @@ extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, + [BINARY_OP_INPLACE_ADD_UNICODE] = 1, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, [SEND] = 1, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 790e6b1b1b91e6..7e79616d41670b 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -138,6 +138,9 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); spam(); DISPATCH(); } @@ -152,6 +155,9 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; spam(); @@ -169,6 +175,9 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *res; spam(); STACK_GROW(1); @@ -186,6 +195,9 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -204,6 +216,9 @@ def test_binary_op(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *right; PyObject *left; PyObject *res; @@ -225,6 +240,9 @@ def test_overlap(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *right; PyObject *left; PyObject *result; @@ -249,6 +267,9 @@ def test_predictions_and_eval_breaker(self): """ output = """ TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); PREDICTED(OP1); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); PyObject *arg; @@ -259,6 +280,9 @@ def test_predictions_and_eval_breaker(self): } TARGET(OP3) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP3); PyObject *arg; PyObject *res; arg = stack_pointer[-1]; @@ -278,6 +302,9 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); if (cond) goto label; DISPATCH(); } @@ -292,6 +319,9 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); if (cond) goto label; DISPATCH(); } @@ -306,6 +336,9 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *right; PyObject *left; PyObject *res; @@ -326,12 +359,14 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; - uint16_t counter = read_u16(&next_instr[0].cache); - uint32_t extra = read_u32(&next_instr[1].cache); + uint16_t counter = read_u16(&here[1].cache); + uint32_t extra = read_u32(&here[2].cache); STACK_SHRINK(1); - next_instr += 3; DISPATCH(); } """ @@ -345,6 +380,9 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); goto somewhere; } """ @@ -366,18 +404,24 @@ def test_macro_instruction(self): """ output = """ TARGET(OP1) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(OP1); PyObject *right; PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - uint16_t counter = read_u16(&next_instr[0].cache); + uint16_t counter = read_u16(&here[1].cache); op1(left, right); - next_instr += 1; DISPATCH(); } TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP); PREDICTED(OP); + _Py_CODEUNIT *here = next_instr - 6; static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; @@ -387,22 +431,24 @@ def test_macro_instruction(self): right = stack_pointer[-1]; left = stack_pointer[-2]; { - uint16_t counter = read_u16(&next_instr[0].cache); + uint16_t counter = read_u16(&here[1].cache); op1(left, right); } // OP2 arg2 = stack_pointer[-3]; { - uint32_t extra = read_u32(&next_instr[3].cache); + uint32_t extra = read_u32(&here[4].cache); res = op2(arg2, left, right); } STACK_SHRINK(2); stack_pointer[-1] = res; - next_instr += 5; DISPATCH(); } TARGET(OP3) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP3); PyObject *right; PyObject *left; PyObject *arg2; @@ -413,7 +459,6 @@ def test_macro_instruction(self): res = op3(arg2, left, right); STACK_SHRINK(2); stack_pointer[-1] = res; - next_instr += 5; DISPATCH(); } """ @@ -427,6 +472,9 @@ def test_array_input(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *above; PyObject **values; PyObject *below; @@ -449,6 +497,9 @@ def test_array_output(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *below; PyObject **values; PyObject *above; @@ -470,6 +521,9 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject **values; PyObject *above; values = stack_pointer - oparg; @@ -489,6 +543,9 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject **values; PyObject *extra; values = stack_pointer - oparg; @@ -509,6 +566,9 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *cc; PyObject *input = NULL; PyObject *aa; @@ -541,6 +601,9 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); PyObject *right; PyObject *middle; PyObject *left; @@ -580,6 +643,9 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); PyObject *val1; PyObject *val2; // A @@ -609,6 +675,9 @@ def test_override_inst(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); ham(); DISPATCH(); } @@ -627,6 +696,9 @@ def test_override_op(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); ham(); DISPATCH(); } diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 3b8ecb765c23f0..d12cd9637dd96e 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1378,15 +1378,16 @@ def func(): x = 4 else: x = 6 + 7 self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ ('branch', 'func', 2, 2), - ('branch', 'func', 3, 4), + ('branch', 'func', 3, 6), ('jump', 'func', 6, 2), ('branch', 'func', 2, 2), - ('branch', 'func', 3, 3), + ('branch', 'func', 3, 4), ('jump', 'func', 4, 2), - ('branch', 'func', 2, 2)]) + ('branch', 'func', 2, 7)]) self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ ('line', 'get_events', 10), @@ -1394,17 +1395,18 @@ def func(): ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), - ('branch', 'func', 3, 4), + ('branch', 'func', 3, 6), ('line', 'func', 6), ('jump', 'func', 6, 2), ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), - ('branch', 'func', 3, 3), + ('branch', 'func', 3, 4), ('line', 'func', 4), ('jump', 'func', 4, 2), ('line', 'func', 2), - ('branch', 'func', 2, 2), + ('branch', 'func', 2, 7), + ('line', 'func', 7), ('line', 'get_events', 11)]) def test_except_star(self): @@ -1434,7 +1436,7 @@ def func(): ('line', 'meth', 1), ('jump', 'func', 5, 5), ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=122]'), + ('branch', 'func', '[offset=120]', '[offset=124]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1450,7 +1452,7 @@ def func(): ('return', None), ('jump', 'func', 5, 5), ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=122]'), + ('branch', 'func', '[offset=120]', '[offset=124]'), ('return', None), ('line', 'get_events', 11)]) diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 82b26f8cbea1a4..232c569a3ddb30 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -438,14 +438,6 @@ break; } - case LOAD_SUPER_ATTR: { - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - case LOAD_SUPER_ATTR_ATTR: { STACK_SHRINK(2); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 493be36f3b28b9..b16edf93d23743 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -80,6 +80,7 @@ dummy_func( pop_1_error: // Dummy locals. PyObject *dummy; + _Py_CODEUNIT *here; PyObject *attr; PyObject *attrs; PyObject *bottom; @@ -144,13 +145,13 @@ dummy_func( if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); - next_instr = frame->instr_ptr; + next_instr = here; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - next_instr[-1].op.code = RESUME_CHECK; + here->op.code = RESUME_CHECK; } } @@ -172,10 +173,9 @@ dummy_func( if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr = frame->instr_ptr; + next_instr = here; } else { - _Py_CODEUNIT *here = frame->instr_ptr; if (oparg < 2) { CHECK_EVAL_BREAKER(); } @@ -186,7 +186,7 @@ dummy_func( ERROR_IF(err, error); if (frame->instr_ptr != here) { /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; + next_instr = here; DISPATCH(); } } @@ -313,14 +313,13 @@ dummy_func( inst(TO_BOOL, (unused/1, unused/2, value -- res)) { #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + next_instr = here; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); DECREF_INPUTS(); @@ -494,10 +493,9 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); + op(_BINARY_OP_INPLACE_ADD_UNICODE, (unused/1, left, right --)) { + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -517,7 +515,8 @@ dummy_func( _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); ERROR_IF(*target_local == NULL, error); // The STORE_FAST is already done. - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); } macro(BINARY_OP_INPLACE_ADD_UNICODE) = @@ -533,14 +532,13 @@ dummy_func( inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) { #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); DECREF_INPUTS(); @@ -659,7 +657,6 @@ dummy_func( STACK_SHRINK(2); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); @@ -682,14 +679,13 @@ dummy_func( inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -961,14 +957,13 @@ dummy_func( inst(SEND, (unused/1, receiver, v -- receiver, retval)) { #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && @@ -982,7 +977,6 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); @@ -1021,7 +1015,6 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); @@ -1030,7 +1023,6 @@ dummy_func( inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - _Py_CODEUNIT *here = frame->instr_ptr; frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1081,7 +1073,6 @@ dummy_func( } inst(RERAISE, (values[oparg], exc -- values[oparg])) { - _Py_CODEUNIT *here = frame->instr_ptr; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1189,14 +1180,13 @@ dummy_func( inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) { #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -1252,15 +1242,14 @@ dummy_func( inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) { #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = frame->instr_ptr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1374,15 +1363,14 @@ dummy_func( inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = frame->instr_ptr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1697,11 +1685,10 @@ dummy_func( ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); } - inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) { - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(here[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -1711,24 +1698,24 @@ dummy_func( }; inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) { + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr = frame->instr_ptr; + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + next_instr = here; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, global_super, arg); + frame, here, global_super, arg); ERROR_IF(err, error); } @@ -1741,12 +1728,12 @@ dummy_func( if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, global_super, arg); + frame, here, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, global_super, arg); + frame, here, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -1824,15 +1811,14 @@ dummy_func( inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = frame->instr_ptr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -2023,8 +2009,7 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2051,7 +2036,6 @@ dummy_func( STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); @@ -2147,14 +2131,13 @@ dummy_func( inst(COMPARE_OP, (unused/1, left, right -- res)) { #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -2282,7 +2265,6 @@ dummy_func( inst(JUMP_BACKWARD, (--)) { CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = frame->instr_ptr; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION @@ -2333,7 +2315,6 @@ dummy_func( } inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION @@ -2343,7 +2324,6 @@ dummy_func( } inst(POP_JUMP_IF_TRUE, (unused/1, cond -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION @@ -2463,14 +2443,13 @@ dummy_func( inst(FOR_ITER, (unused/1, iter -- iter, next)) { #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2483,11 +2462,10 @@ dummy_func( _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -2495,14 +2473,13 @@ dummy_func( // Common case: no jump, leave it to the code generator } - inst(INSTRUMENTED_FOR_ITER, ( -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; + inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + target = next_instr; } else { if (_PyErr_Occurred(tstate)) { @@ -2513,12 +2490,12 @@ dummy_func( _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + target = next_instr + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); } @@ -2539,7 +2516,6 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -2595,7 +2571,6 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -2647,7 +2622,6 @@ dummy_func( if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); // Jump over END_FOR instruction. JUMPBY(oparg + 1); DISPATCH(); @@ -2689,7 +2663,6 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); @@ -2919,7 +2892,7 @@ dummy_func( unused/2 + _LOAD_ATTR_METHOD_LAZY_DICT; - inst(INSTRUMENTED_CALL, ( -- )) { + inst(INSTRUMENTED_CALL, (unused/3 -- )) { int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -2927,10 +2900,9 @@ dummy_func( &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, function, arg); + frame, here, function, arg); ERROR_IF(err, error); - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(here[1].cache); GO_TO_INSTRUCTION(CALL); } @@ -2966,14 +2938,13 @@ dummy_func( total_args++; } #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; @@ -3003,7 +2974,6 @@ dummy_func( if (new_frame == NULL) { goto error; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); @@ -3158,7 +3128,6 @@ dummy_func( } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); @@ -3207,7 +3176,7 @@ dummy_func( * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)next_instr; + _PyCallCache *cache = (_PyCallCache *)&frame->instr_ptr[1]; DEOPT_IF(null != NULL); DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; @@ -3236,7 +3205,6 @@ dummy_func( for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); @@ -3433,9 +3401,9 @@ dummy_func( Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); - // CALL + POP_TOP - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } @@ -3565,7 +3533,7 @@ dummy_func( : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr - 1, function, arg); + frame, frame->instr_ptr, function, arg); ERROR_IF(err, error); GO_TO_INSTRUCTION(CALL_KW); } @@ -3825,14 +3793,13 @@ dummy_func( inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -3849,12 +3816,11 @@ dummy_func( inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, frame->instr_ptr); + tstate, frame, here); ERROR_IF(next_opcode < 0, error); - next_instr = frame->instr_ptr; + next_instr = here; if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(here[1].cache); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -3862,12 +3828,10 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); } @@ -3875,7 +3839,6 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3887,7 +3850,6 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3898,7 +3860,6 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -3916,7 +3877,6 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { diff --git a/Python/ceval.c b/Python/ceval.c index e3a7c5f38403a7..065085278b369e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -866,7 +866,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int monitor_raise(tstate, frame, next_instr-1); exception_unwind: { - /* We can't use frame->f_lasti here, as RERAISE may have set it */ + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 544e8ef8fa8c0a..b11240b1faed49 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -60,25 +60,21 @@ #endif #ifdef Py_STATS -#define INSTRUCTION_START(op) \ +#define INSTRUCTION_STATS(op) \ do { \ - frame->instr_ptr = next_instr++; \ OPCODE_EXE_INC(op); \ if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else -#define INSTRUCTION_START(op) \ - do { \ - frame->instr_ptr = next_instr++; \ - } while(0) +#define INSTRUCTION_STATS(op) ((void)0) #endif #if USE_COMPUTED_GOTOS -# define TARGET(op) TARGET_##op: INSTRUCTION_START(op); +# define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] #else -# define TARGET(op) case op: TARGET_##op: INSTRUCTION_START(op); +# define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode #endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0c52afaec981e5..c61b2e5036b480 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -119,14 +119,13 @@ PyObject *res; value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + next_instr = here; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); @@ -380,14 +379,13 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); @@ -574,14 +572,13 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -900,14 +897,13 @@ PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -989,15 +985,14 @@ owner = stack_pointer[-1]; v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = frame->instr_ptr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1133,15 +1128,14 @@ PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = frame->instr_ptr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1559,71 +1553,6 @@ break; } - case LOAD_SUPER_ATTR: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *null = NULL; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; - #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr = frame->instr_ptr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, global_super, arg); - if (err) goto pop_3_error; - } - - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } - } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - break; - } - case LOAD_SUPER_ATTR_ATTR: { PyObject *self; PyObject *class; @@ -1689,15 +1618,14 @@ PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = frame->instr_ptr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -1952,14 +1880,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -3274,14 +3201,13 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 78a91f9c4d7400..c1f3ac91e0a48d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4,11 +4,18 @@ // Do not edit! TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); DISPATCH(); } TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); + _Py_CODEUNIT *here = next_instr - 1; static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); @@ -20,18 +27,21 @@ if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; - next_instr = frame->instr_ptr; + next_instr = here; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - next_instr[-1].op.code = RESUME_CHECK; + here->op.code = RESUME_CHECK; } DISPATCH(); } TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -44,16 +54,18 @@ } TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr = frame->instr_ptr; + next_instr = here; } else { - _Py_CODEUNIT *here = frame->instr_ptr; if (oparg < 2) { CHECK_EVAL_BREAKER(); } @@ -64,7 +76,7 @@ if (err) goto error; if (frame->instr_ptr != here) { /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; + next_instr = here; DISPATCH(); } } @@ -72,6 +84,9 @@ } TARGET(LOAD_FAST_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; @@ -82,6 +97,9 @@ } TARGET(LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); PyObject *value; value = GETLOCAL(oparg); assert(value != NULL); @@ -92,6 +110,9 @@ } TARGET(LOAD_FAST_AND_CLEAR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyObject *value; value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value @@ -102,6 +123,9 @@ } TARGET(LOAD_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; uint32_t oparg1 = oparg >> 4; @@ -117,6 +141,9 @@ } TARGET(LOAD_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); PyObject *value; value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); @@ -126,6 +153,9 @@ } TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); PyObject *value; value = stack_pointer[-1]; SETLOCAL(oparg, value); @@ -134,6 +164,9 @@ } TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; value1 = stack_pointer[-1]; @@ -147,6 +180,9 @@ } TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); PyObject *value1; PyObject *value2; value1 = stack_pointer[-1]; @@ -160,6 +196,9 @@ } TARGET(POP_TOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); @@ -168,6 +207,9 @@ } TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); PyObject *res; res = NULL; STACK_GROW(1); @@ -176,6 +218,9 @@ } TARGET(END_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); PyObject *value; // POP_TOP value = stack_pointer[-1]; @@ -192,6 +237,9 @@ } TARGET(INSTRUMENTED_END_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; @@ -213,6 +261,9 @@ } TARGET(END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; @@ -224,6 +275,9 @@ } TARGET(INSTRUMENTED_END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; @@ -243,6 +297,9 @@ } TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -254,6 +311,9 @@ } TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -264,40 +324,47 @@ } TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); + _Py_CODEUNIT *here = next_instr - 4; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + next_instr = here; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); if (err < 0) goto pop_1_error; res = err ? Py_True : Py_False; stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_BOOL) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); PyObject *value; value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -312,11 +379,13 @@ res = Py_True; } stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_LIST) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -325,11 +394,13 @@ res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_NONE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -338,11 +409,13 @@ STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_STR) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -358,15 +431,17 @@ res = Py_True; } stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = read_u32(&next_instr[1].cache); + uint32_t version = read_u32(&here[2].cache); // This one is a bit weird, because we expect *some* failures: assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); @@ -374,11 +449,13 @@ Py_DECREF(value); res = Py_True; stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -390,6 +467,9 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); PyObject *right; PyObject *left; PyObject *res; @@ -410,11 +490,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); PyObject *right; PyObject *left; PyObject *res; @@ -435,11 +517,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); PyObject *right; PyObject *left; PyObject *res; @@ -460,11 +544,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -485,11 +571,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -510,11 +598,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -535,11 +625,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); PyObject *right; PyObject *left; PyObject *res; @@ -560,11 +652,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -576,9 +670,8 @@ } // _BINARY_OP_INPLACE_ADD_UNICODE { - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -598,14 +691,19 @@ _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); } STACK_SHRINK(2); DISPATCH(); } TARGET(BINARY_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; @@ -613,14 +711,13 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); @@ -628,11 +725,13 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); PyObject *stop; PyObject *start; PyObject *container; @@ -658,6 +757,9 @@ } TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); PyObject *stop; PyObject *start; PyObject *container; @@ -683,6 +785,9 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *res; @@ -703,11 +808,13 @@ Py_DECREF(list); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_STR_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); PyObject *sub; PyObject *str; PyObject *res; @@ -727,11 +834,13 @@ Py_DECREF(str); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_TUPLE_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); PyObject *sub; PyObject *tuple; PyObject *res; @@ -752,11 +861,13 @@ Py_DECREF(tuple); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_DICT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *res; @@ -778,11 +889,13 @@ Py_DECREF(sub); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -806,13 +919,15 @@ STACK_SHRINK(2); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } TARGET(LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); PyObject *v; PyObject *list; v = stack_pointer[-1]; @@ -823,6 +938,9 @@ } TARGET(SET_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); PyObject *v; PyObject *set; v = stack_pointer[-1]; @@ -835,7 +953,11 @@ } TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; @@ -844,14 +966,13 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -860,11 +981,13 @@ Py_DECREF(sub); if (err) goto pop_3_error; STACK_SHRINK(3); - next_instr += 1; DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *value; @@ -888,11 +1011,13 @@ _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); STACK_SHRINK(3); - next_instr += 1; DISPATCH(); } TARGET(STORE_SUBSCR_DICT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *value; @@ -905,11 +1030,13 @@ Py_DECREF(dict); if (err) goto pop_3_error; STACK_SHRINK(3); - next_instr += 1; DISPATCH(); } TARGET(DELETE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -924,6 +1051,9 @@ } TARGET(CALL_INTRINSIC_1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -936,6 +1066,9 @@ } TARGET(CALL_INTRINSIC_2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); PyObject *value1; PyObject *value2; PyObject *res; @@ -952,6 +1085,9 @@ } TARGET(RAISE_VARARGS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; args = stack_pointer - oparg; PyObject *cause = NULL, *exc = NULL; @@ -978,6 +1114,9 @@ } TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); PyObject *retval; retval = stack_pointer[-1]; assert(frame == &entry_frame); @@ -990,6 +1129,9 @@ } TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; STACK_SHRINK(1); @@ -1016,6 +1158,9 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( @@ -1037,6 +1182,9 @@ } TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); PyObject *value; PyObject *retval; // LOAD_CONST @@ -1071,6 +1219,9 @@ } TARGET(INSTRUMENTED_RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -1091,6 +1242,9 @@ } TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); PyObject *obj; PyObject *iter; obj = stack_pointer[-1]; @@ -1129,6 +1283,9 @@ } TARGET(GET_ANEXT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); PyObject *aiter; PyObject *awaitable; aiter = stack_pointer[-1]; @@ -1180,6 +1337,9 @@ } TARGET(GET_AWAITABLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -1211,7 +1371,11 @@ } TARGET(SEND) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); PREDICTED(SEND); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; @@ -1219,14 +1383,13 @@ v = stack_pointer[-1]; receiver = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && @@ -1240,7 +1403,6 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); @@ -1266,11 +1428,13 @@ } Py_DECREF(v); stack_pointer[-1] = retval; - next_instr += 1; DISPATCH(); } TARGET(SEND_GEN) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; @@ -1286,18 +1450,19 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - _Py_CODEUNIT *here = frame->instr_ptr; frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1320,6 +1485,9 @@ } TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -1345,6 +1513,9 @@ } TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -1354,11 +1525,13 @@ } TARGET(RERAISE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); PyObject *exc; PyObject **values; exc = stack_pointer[-1]; values = stack_pointer - 1 - oparg; - _Py_CODEUNIT *here = frame->instr_ptr; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1380,6 +1553,9 @@ } TARGET(END_ASYNC_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); PyObject *exc; PyObject *awaitable; exc = stack_pointer[-1]; @@ -1400,6 +1576,9 @@ } TARGET(CLEANUP_THROW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); PyObject *exc_value; PyObject *last_sent_val; PyObject *sub_iter; @@ -1429,6 +1608,9 @@ } TARGET(LOAD_ASSERTION_ERROR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; value = Py_NewRef(PyExc_AssertionError); STACK_GROW(1); @@ -1437,6 +1619,9 @@ } TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyObject *bc; if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; if (bc == NULL) { @@ -1450,6 +1635,9 @@ } TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); PyObject *v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1472,6 +1660,9 @@ } TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -1492,19 +1683,22 @@ } TARGET(UNPACK_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -1512,11 +1706,13 @@ if (res == 0) goto pop_1_error; STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1530,11 +1726,13 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_TUPLE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1549,11 +1747,13 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_LIST) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1568,11 +1768,13 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); PyObject *seq; seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); @@ -1585,22 +1787,25 @@ } TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); + _Py_CODEUNIT *here = next_instr - 5; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; owner = stack_pointer[-1]; v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = frame->instr_ptr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1608,11 +1813,13 @@ Py_DECREF(owner); if (err) goto pop_2_error; STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); PyObject *owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1624,6 +1831,9 @@ } TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); PyObject *v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1635,6 +1845,9 @@ } TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1650,6 +1863,9 @@ } TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); PyObject *locals; locals = LOCALS(); if (locals == NULL) { @@ -1664,6 +1880,9 @@ } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); PyObject *mod_or_class_dict; PyObject *v; mod_or_class_dict = stack_pointer[-1]; @@ -1697,6 +1916,9 @@ } TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); PyObject *v; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { @@ -1734,20 +1956,23 @@ } TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *here = next_instr - 5; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = frame->instr_ptr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1788,16 +2013,18 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { - uint16_t version = read_u16(&next_instr[1].cache); + uint16_t version = read_u16(&here[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1805,7 +2032,7 @@ } // _LOAD_GLOBAL_MODULE { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -1818,16 +2045,18 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { - uint16_t version = read_u16(&next_instr[1].cache); + uint16_t version = read_u16(&here[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1835,7 +2064,7 @@ } // _GUARD_BUILTINS_VERSION { - uint16_t version = read_u16(&next_instr[2].cache); + uint16_t version = read_u16(&here[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1843,7 +2072,7 @@ } // _LOAD_GLOBAL_BUILTINS { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -1856,11 +2085,13 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; DISPATCH(); } TARGET(DELETE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); @@ -1868,6 +2099,9 @@ } TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1880,6 +2114,9 @@ } TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1894,6 +2131,9 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); PyObject *class_dict; PyObject *value; class_dict = stack_pointer[-1]; @@ -1920,6 +2160,9 @@ } TARGET(LOAD_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); @@ -1934,6 +2177,9 @@ } TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); PyObject *v; v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); @@ -1945,6 +2191,9 @@ } TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyFunction_Check(frame->f_funcobj)); @@ -1959,6 +2208,9 @@ } TARGET(BUILD_STRING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); PyObject **pieces; PyObject *str; pieces = stack_pointer - oparg; @@ -1974,6 +2226,9 @@ } TARGET(BUILD_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); PyObject **values; PyObject *tup; values = stack_pointer - oparg; @@ -1986,6 +2241,9 @@ } TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); PyObject **values; PyObject *list; values = stack_pointer - oparg; @@ -1998,6 +2256,9 @@ } TARGET(LIST_EXTEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); PyObject *iterable; PyObject *list; iterable = stack_pointer[-1]; @@ -2022,6 +2283,9 @@ } TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); PyObject *iterable; PyObject *set; iterable = stack_pointer[-1]; @@ -2034,6 +2298,9 @@ } TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); PyObject **values; PyObject *set; values = stack_pointer - oparg; @@ -2058,6 +2325,9 @@ } TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); PyObject **values; PyObject *map; values = stack_pointer - oparg*2; @@ -2076,6 +2346,9 @@ } TARGET(SETUP_ANNOTATIONS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2117,6 +2390,9 @@ } TARGET(BUILD_CONST_KEY_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); PyObject *keys; PyObject **values; PyObject *map; @@ -2142,6 +2418,9 @@ } TARGET(DICT_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); PyObject *update; PyObject *dict; update = stack_pointer[-1]; @@ -2161,6 +2440,9 @@ } TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); PyObject *update; PyObject *dict; PyObject *callable; @@ -2178,6 +2460,9 @@ } TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); PyObject *value; PyObject *key; PyObject *dict; @@ -2193,15 +2478,21 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(here[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; @@ -2211,24 +2502,24 @@ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr = frame->instr_ptr; + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + next_instr = here; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, global_super, arg); + frame, here, global_super, arg); if (err) goto pop_3_error; } @@ -2241,12 +2532,12 @@ if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, global_super, arg); + frame, here, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, global_super, arg); + frame, here, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -2264,11 +2555,13 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); PyObject *self; PyObject *class; PyObject *global_super; @@ -2288,11 +2581,13 @@ if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); stack_pointer[-1] = attr; - next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); PyObject *self; PyObject *class; PyObject *global_super; @@ -2325,27 +2620,29 @@ STACK_SHRINK(1); stack_pointer[-2] = attr; stack_pointer[-1] = self_or_null; - next_instr += 1; DISPATCH(); } TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *here = next_instr - 10; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = frame->instr_ptr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -2382,18 +2679,20 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2407,7 +2706,7 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -2419,18 +2718,20 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2438,7 +2739,7 @@ } // _LOAD_ATTR_MODULE { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); @@ -2453,18 +2754,20 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2480,7 +2783,7 @@ } // _LOAD_ATTR_WITH_HINT { - uint16_t hint = read_u16(&next_instr[3].cache); + uint16_t hint = read_u16(&here[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); @@ -2504,25 +2807,27 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -2534,25 +2839,27 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&here[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2562,16 +2869,18 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *fget = read_obj(&next_instr[5].cache); + uint32_t type_version = read_u32(&here[2].cache); + uint32_t func_version = read_u32(&here[4].cache); + PyObject *fget = read_obj(&here[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); @@ -2591,18 +2900,20 @@ // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *getattribute = read_obj(&next_instr[5].cache); + uint32_t type_version = read_u32(&here[2].cache); + uint32_t func_version = read_u32(&here[4].cache); + PyObject *getattribute = read_obj(&here[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2624,19 +2935,21 @@ STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2650,7 +2963,7 @@ // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); PyDictValues *values = _PyDictOrValues_GetValues(dorv); @@ -2665,17 +2978,19 @@ Py_DECREF(owner); } STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t hint = read_u16(&next_instr[3].cache); + uint32_t type_version = read_u32(&here[2].cache); + uint16_t hint = read_u16(&here[4].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2715,17 +3030,19 @@ dict->ma_version_tag = new_version; Py_DECREF(owner); STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2733,7 +3050,7 @@ // _STORE_ATTR_SLOT value = stack_pointer[-2]; { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&here[4].cache); char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -2742,12 +3059,15 @@ Py_DECREF(owner); } STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; @@ -2755,14 +3075,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -2777,11 +3096,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(COMPARE_OP_FLOAT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -2800,11 +3121,13 @@ // It's always a bool, so we don't care about oparg & 16. STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(COMPARE_OP_INT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); PyObject *right; PyObject *left; PyObject *res; @@ -2827,11 +3150,13 @@ // It's always a bool, so we don't care about oparg & 16. STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(COMPARE_OP_STR) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); PyObject *right; PyObject *left; PyObject *res; @@ -2851,11 +3176,13 @@ // It's always a bool, so we don't care about oparg & 16. STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); PyObject *right; PyObject *left; PyObject *b; @@ -2871,6 +3198,9 @@ } TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONTAINS_OP); PyObject *right; PyObject *left; PyObject *b; @@ -2887,6 +3217,9 @@ } TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); PyObject *match_type; PyObject *exc_value; PyObject *rest; @@ -2919,6 +3252,9 @@ } TARGET(CHECK_EXC_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); PyObject *right; PyObject *left; PyObject *b; @@ -2938,6 +3274,9 @@ } TARGET(IMPORT_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); PyObject *fromlist; PyObject *level; PyObject *res; @@ -2954,6 +3293,9 @@ } TARGET(IMPORT_FROM) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); PyObject *from; PyObject *res; from = stack_pointer[-1]; @@ -2966,13 +3308,18 @@ } TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); } TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD); CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION @@ -2996,6 +3343,9 @@ } TARGET(ENTER_EXECUTOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); @@ -3014,9 +3364,11 @@ } TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; cond = stack_pointer[-1]; - _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION @@ -3024,14 +3376,15 @@ #endif JUMPBY(oparg * flag); STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; cond = stack_pointer[-1]; - _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION @@ -3039,11 +3392,13 @@ #endif JUMPBY(oparg * flag); STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyObject *value; PyObject *b; PyObject *cond; @@ -3061,7 +3416,6 @@ // POP_JUMP_IF_TRUE cond = b; { - _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION @@ -3070,11 +3424,13 @@ JUMPBY(oparg * flag); } STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyObject *value; PyObject *b; PyObject *cond; @@ -3092,7 +3448,6 @@ // POP_JUMP_IF_FALSE cond = b; { - _Py_CODEUNIT *here = frame->instr_ptr; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION @@ -3101,11 +3456,13 @@ JUMPBY(oparg * flag); } STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3116,6 +3473,9 @@ } TARGET(GET_LEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; @@ -3130,6 +3490,9 @@ } TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); PyObject *names; PyObject *type; PyObject *subject; @@ -3157,6 +3520,9 @@ } TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); PyObject *subject; PyObject *res; subject = stack_pointer[-1]; @@ -3168,6 +3534,9 @@ } TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); PyObject *subject; PyObject *res; subject = stack_pointer[-1]; @@ -3179,6 +3548,9 @@ } TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); PyObject *keys; PyObject *subject; PyObject *values_or_none; @@ -3193,6 +3565,9 @@ } TARGET(GET_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -3205,6 +3580,9 @@ } TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -3237,20 +3615,23 @@ } TARGET(FOR_ITER) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3263,11 +3644,10 @@ _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3275,18 +3655,19 @@ // Common case: no jump, leave it to the code generator STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *here = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + target = next_instr; } else { if (_PyErr_Occurred(tstate)) { @@ -3297,18 +3678,21 @@ _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + target = next_instr + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(FOR_ITER_LIST) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); PyObject *iter; PyObject *next; // _ITER_CHECK_LIST @@ -3329,7 +3713,6 @@ } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3346,11 +3729,13 @@ } STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); PyObject *iter; PyObject *next; // _ITER_CHECK_TUPLE @@ -3371,7 +3756,6 @@ } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3388,11 +3772,13 @@ } STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_RANGE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); PyObject *iter; PyObject *next; // _ITER_CHECK_RANGE @@ -3409,7 +3795,6 @@ if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); // Jump over END_FOR instruction. JUMPBY(oparg + 1); DISPATCH(); @@ -3428,11 +3813,13 @@ } STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_GEN) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); @@ -3445,7 +3832,6 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); @@ -3454,6 +3840,9 @@ } TARGET(BEFORE_ASYNC_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); PyObject *mgr; PyObject *exit; PyObject *res; @@ -3494,6 +3883,9 @@ } TARGET(BEFORE_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_WITH); PyObject *mgr; PyObject *exit; PyObject *res; @@ -3537,6 +3929,9 @@ } TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); PyObject *val; PyObject *lasti; PyObject *exit_func; @@ -3575,6 +3970,9 @@ } TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; @@ -3594,13 +3992,16 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; PyObject *attr; PyObject *self; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -3613,14 +4014,14 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&next_instr[3].cache); + uint32_t keys_version = read_u32(&here[4].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&here[6].cache); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3632,25 +4033,27 @@ STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; PyObject *attr; PyObject *self; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&here[6].cache); assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3662,17 +4065,19 @@ STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; PyObject *attr; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -3685,14 +4090,14 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&next_instr[3].cache); + uint32_t keys_version = read_u32(&here[4].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&here[6].cache); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3700,24 +4105,26 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyObject *owner; PyObject *attr; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&here[6].cache); assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3726,18 +4133,20 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); PyObject *owner; PyObject *attr; PyObject *self; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&here[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -3752,7 +4161,7 @@ } // _LOAD_ATTR_METHOD_LAZY_DICT { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&here[6].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3763,11 +4172,13 @@ STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; - next_instr += 9; DISPATCH(); } TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -3775,15 +4186,18 @@ &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, function, arg); + frame, here, function, arg); if (err) goto error; - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(here[1].cache); GO_TO_INSTRUCTION(CALL); } TARGET(CALL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); PREDICTED(CALL); + _Py_CODEUNIT *here = next_instr - 4; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; @@ -3799,14 +4213,13 @@ total_args++; } #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; @@ -3836,7 +4249,6 @@ if (new_frame == NULL) { goto error; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); @@ -3872,12 +4284,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); PyObject *null; PyObject *callable; PyObject *self; @@ -3909,7 +4323,7 @@ self_or_null = self; callable = func; { - uint32_t func_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&here[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -3939,7 +4353,6 @@ } } // _SAVE_RETURN_OFFSET - next_instr += 3; { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); @@ -3973,6 +4386,9 @@ } TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); PyObject *self_or_null; PyObject *callable; PyObject **args; @@ -3985,7 +4401,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - uint32_t func_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&here[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -4015,7 +4431,6 @@ } } // _SAVE_RETURN_OFFSET - next_instr += 3; { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); @@ -4049,13 +4464,16 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); PyObject **args; PyObject *self_or_null; PyObject *callable; args = stack_pointer - oparg; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&here[2].cache); DEOPT_IF(tstate->interp->eval_frame, CALL); int argcount = oparg; if (self_or_null != NULL) { @@ -4085,13 +4503,15 @@ } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } TARGET(CALL_TYPE_1) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4110,11 +4530,13 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(CALL_STR_1) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4134,12 +4556,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_TUPLE_1) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4159,12 +4583,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; PyObject *null; PyObject *callable; @@ -4176,7 +4602,7 @@ * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)next_instr; + _PyCallCache *cache = (_PyCallCache *)&frame->instr_ptr[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -4205,7 +4631,6 @@ for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); @@ -4223,6 +4648,9 @@ } TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); PyObject *should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); @@ -4237,6 +4665,9 @@ } TARGET(CALL_BUILTIN_CLASS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4263,12 +4694,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_O) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4303,12 +4736,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_FAST) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4347,12 +4782,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4385,12 +4822,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_LEN) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4422,11 +4861,13 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(CALL_ISINSTANCE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4460,11 +4901,13 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(CALL_LIST_APPEND) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); PyObject **args; PyObject *self; PyObject *callable; @@ -4483,13 +4926,16 @@ Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); - // CALL + POP_TOP - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_O) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4527,12 +4973,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4568,12 +5016,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4610,12 +5060,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4650,12 +5102,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_CALL_KW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); int is_meth = PEEK(oparg + 2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 3); @@ -4663,12 +5117,15 @@ : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr - 1, function, arg); + frame, frame->instr_ptr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } TARGET(CALL_KW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); PyObject *kwnames; PyObject **args; @@ -4756,10 +5213,16 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); PyObject *kwargs = NULL; PyObject *callargs; @@ -4843,6 +5306,9 @@ } TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; @@ -4863,6 +5329,9 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); PyObject *func; PyObject *attr; func = stack_pointer[-1]; @@ -4897,6 +5366,9 @@ } TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4922,6 +5394,9 @@ } TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); PyObject *step = NULL; PyObject *stop; PyObject *start; @@ -4941,6 +5416,9 @@ } TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); PyObject *value; PyObject *result; value = stack_pointer[-1]; @@ -4955,6 +5433,9 @@ } TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -4973,6 +5454,9 @@ } TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); PyObject *fmt_spec; PyObject *value; PyObject *res; @@ -4988,6 +5472,9 @@ } TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); PyObject *bottom; PyObject *top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -4999,7 +5486,11 @@ } TARGET(BINARY_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); + _Py_CODEUNIT *here = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; @@ -5007,14 +5498,13 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(here[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -5025,11 +5515,13 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); PyObject *top; PyObject *bottom; top = stack_pointer[-1]; @@ -5041,13 +5533,15 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, frame->instr_ptr); + tstate, frame, here); if (next_opcode < 0) goto error; - next_instr = frame->instr_ptr; + next_instr = here; if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(here[1].cache); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -5055,49 +5549,57 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *here = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *here = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -5111,13 +5613,14 @@ here[1].cache = (here[1].cache << 1) | flag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { @@ -5131,11 +5634,13 @@ here[1].cache = (here[1].cache << 1) | !nflag; #endif INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; DISPATCH(); } TARGET(EXTENDED_ARG) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -5144,11 +5649,17 @@ } TARGET(CACHE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_UNREACHABLE(); } TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 53f715db6f24a5..180bd01302ec11 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -432,7 +432,6 @@ def report_non_viable_uops(self, jsonfile: str) -> None: "LOAD_FAST_LOAD_FAST", "LOAD_CONST_LOAD_FAST", "STORE_FAST_STORE_FAST", - "_BINARY_OP_INPLACE_ADD_UNICODE", "POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE", "_ITER_JUMP_LIST", diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index c6b551675e3e7e..b70f1b1aa0c931 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -61,6 +61,7 @@ class Instruction: # Computed by constructor always_exits: str # If the block always exits, its last line; else "" has_deopt: bool + needs_here: bool cache_offset: int cache_effects: list[parsing.CacheEffect] input_effects: list[StackEffect] @@ -83,6 +84,7 @@ def __init__(self, inst: parsing.InstDef): ) self.always_exits = always_exits(self.block_text) self.has_deopt = variable_used(self.inst, "DEOPT_IF") + self.needs_here = variable_used(self.inst, "here") self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] @@ -164,7 +166,8 @@ def write_body( func = f"read_u{bits}" if tier == TIER_ONE: out.emit( - f"{typ}{ceffect.name} = {func}(&next_instr[{active.offset}].cache);" + f"{typ}{ceffect.name} = " + f"{func}(&here[{active.offset + 1}].cache);" ) else: out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 69a6c10d6199c8..86113604d341e5 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -365,8 +365,17 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: ] out.emit("") with out.block(f"TARGET({mac.name})"): + needs_here = any(part.instr.needs_here for part in parts) or mac.cache_offset != 0 + if needs_here and not mac.predicted: + out.emit(f"_Py_CODEUNIT *here = frame->instr_ptr = next_instr;") + else: + out.emit(f"frame->instr_ptr = next_instr;") + out.emit(f"next_instr += {mac.cache_offset+1};") + out.emit(f"INSTRUCTION_STATS({mac.name});") if mac.predicted: out.emit(f"PREDICTED({mac.name});") + if needs_here: + out.emit(f"_Py_CODEUNIT *here = next_instr - {mac.cache_offset+1};") out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) try: next_instr_is_set = write_components( @@ -375,8 +384,6 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err if not parts[-1].instr.always_exits: - if not next_instr_is_set and mac.cache_offset: - out.emit(f"next_instr += {mac.cache_offset};") if parts[-1].instr.check_eval_breaker: out.emit("CHECK_EVAL_BREAKER();") out.emit("DISPATCH();") @@ -450,8 +457,6 @@ def write_components( if mgr.instr.name == "_SAVE_RETURN_OFFSET": next_instr_is_set = True - if cache_offset: - out.emit(f"next_instr += {cache_offset};") if tier == TIER_ONE: assert_no_pokes(managers) From a23102c522821f483017471372b52d8590afca70 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Oct 2023 09:52:13 +0000 Subject: [PATCH 3/6] Don't define 'here' unless it is used --- Python/generated_cases.c.h | 96 +++++++++++++-------------- Tools/cases_generator/instructions.py | 2 +- Tools/cases_generator/stacking.py | 2 +- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c1f3ac91e0a48d..de939bdcd6f3d7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -351,7 +351,7 @@ } TARGET(TO_BOOL_BOOL) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); PyObject *value; @@ -362,7 +362,7 @@ } TARGET(TO_BOOL_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); PyObject *value; @@ -383,7 +383,7 @@ } TARGET(TO_BOOL_LIST) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); PyObject *value; @@ -398,7 +398,7 @@ } TARGET(TO_BOOL_NONE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); PyObject *value; @@ -413,7 +413,7 @@ } TARGET(TO_BOOL_STR) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); PyObject *value; @@ -467,7 +467,7 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); PyObject *right; @@ -494,7 +494,7 @@ } TARGET(BINARY_OP_ADD_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); PyObject *right; @@ -521,7 +521,7 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); PyObject *right; @@ -548,7 +548,7 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); PyObject *right; @@ -575,7 +575,7 @@ } TARGET(BINARY_OP_ADD_FLOAT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); PyObject *right; @@ -602,7 +602,7 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); PyObject *right; @@ -629,7 +629,7 @@ } TARGET(BINARY_OP_ADD_UNICODE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); PyObject *right; @@ -656,7 +656,7 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); PyObject *right; @@ -785,7 +785,7 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); PyObject *sub; @@ -812,7 +812,7 @@ } TARGET(BINARY_SUBSCR_STR_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); PyObject *sub; @@ -838,7 +838,7 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); PyObject *sub; @@ -865,7 +865,7 @@ } TARGET(BINARY_SUBSCR_DICT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); PyObject *sub; @@ -893,7 +893,7 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); PyObject *sub; @@ -985,7 +985,7 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); PyObject *sub; @@ -1015,7 +1015,7 @@ } TARGET(STORE_SUBSCR_DICT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); PyObject *sub; @@ -1432,7 +1432,7 @@ } TARGET(SEND_GEN) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); PyObject *v; @@ -1710,7 +1710,7 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); PyObject *seq; @@ -1730,7 +1730,7 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); PyObject *seq; @@ -1751,7 +1751,7 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); PyObject *seq; @@ -2559,7 +2559,7 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); PyObject *self; @@ -2585,7 +2585,7 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); PyObject *self; @@ -3100,7 +3100,7 @@ } TARGET(COMPARE_OP_FLOAT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); PyObject *right; @@ -3125,7 +3125,7 @@ } TARGET(COMPARE_OP_INT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); PyObject *right; @@ -3154,7 +3154,7 @@ } TARGET(COMPARE_OP_STR) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); PyObject *right; @@ -3690,7 +3690,7 @@ } TARGET(FOR_ITER_LIST) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); PyObject *iter; @@ -3733,7 +3733,7 @@ } TARGET(FOR_ITER_TUPLE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); PyObject *iter; @@ -3776,7 +3776,7 @@ } TARGET(FOR_ITER_RANGE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); PyObject *iter; @@ -3817,7 +3817,7 @@ } TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; @@ -4509,7 +4509,7 @@ } TARGET(CALL_TYPE_1) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); PyObject **args; @@ -4534,7 +4534,7 @@ } TARGET(CALL_STR_1) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); PyObject **args; @@ -4561,7 +4561,7 @@ } TARGET(CALL_TUPLE_1) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); PyObject **args; @@ -4588,7 +4588,7 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; @@ -4665,7 +4665,7 @@ } TARGET(CALL_BUILTIN_CLASS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); PyObject **args; @@ -4699,7 +4699,7 @@ } TARGET(CALL_BUILTIN_O) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); PyObject **args; @@ -4741,7 +4741,7 @@ } TARGET(CALL_BUILTIN_FAST) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); PyObject **args; @@ -4787,7 +4787,7 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyObject **args; @@ -4827,7 +4827,7 @@ } TARGET(CALL_LEN) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); PyObject **args; @@ -4865,7 +4865,7 @@ } TARGET(CALL_ISINSTANCE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); PyObject **args; @@ -4905,7 +4905,7 @@ } TARGET(CALL_LIST_APPEND) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); PyObject **args; @@ -4933,7 +4933,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); PyObject **args; @@ -4978,7 +4978,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyObject **args; @@ -5021,7 +5021,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); PyObject **args; @@ -5065,7 +5065,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); PyObject **args; diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index b70f1b1aa0c931..93e31c3d56e924 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -84,11 +84,11 @@ def __init__(self, inst: parsing.InstDef): ) self.always_exits = always_exits(self.block_text) self.has_deopt = variable_used(self.inst, "DEOPT_IF") - self.needs_here = variable_used(self.inst, "here") self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) + self.needs_here = variable_used(self.inst, "here") or any(c.name != UNUSED for c in self.cache_effects) self.input_effects = [ effect for effect in inst.inputs if isinstance(effect, StackEffect) ] diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 86113604d341e5..fea5bdcb82a26d 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -365,7 +365,7 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: ] out.emit("") with out.block(f"TARGET({mac.name})"): - needs_here = any(part.instr.needs_here for part in parts) or mac.cache_offset != 0 + needs_here = any(part.instr.needs_here for part in parts) if needs_here and not mac.predicted: out.emit(f"_Py_CODEUNIT *here = frame->instr_ptr = next_instr;") else: From bdacb4e5b939702b11a61f93e8fcbe8a845060ca Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Oct 2023 10:04:02 +0000 Subject: [PATCH 4/6] Change a few 'frame->instr_ptr' to 'here' --- Lib/test/test_generated_cases.py | 2 +- Python/bytecodes.c | 30 ++++++++++++------------- Python/executor_cases.c.h | 8 +++---- Python/generated_cases.c.h | 38 +++++++++++++++++--------------- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 7e79616d41670b..6ce7876c2a19f7 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -446,7 +446,7 @@ def test_macro_instruction(self): } TARGET(OP3) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP3); PyObject *right; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b16edf93d23743..273d44559c3871 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1813,7 +1813,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2036,7 +2036,7 @@ dummy_func( STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2132,7 +2132,7 @@ dummy_func( inst(COMPARE_OP, (unused/1, left, right -- res)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -2444,7 +2444,7 @@ dummy_func( inst(FOR_ITER, (unused/1, iter -- iter, next)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -2458,7 +2458,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, frame->instr_ptr); + monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2665,7 +2665,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -2939,7 +2939,7 @@ dummy_func( } #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } @@ -2974,7 +2974,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3128,7 +3128,7 @@ dummy_func( } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3176,7 +3176,7 @@ dummy_func( * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)&frame->instr_ptr[1]; + _PyCallCache *cache = (_PyCallCache *)&here[1]; DEOPT_IF(null != NULL); DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; @@ -3205,7 +3205,7 @@ dummy_func( for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3575,7 +3575,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - here == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -3672,7 +3672,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - here == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -3794,7 +3794,7 @@ dummy_func( inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } @@ -3936,7 +3936,7 @@ dummy_func( op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - here); #endif #if TIER_TWO frame->return_offset = oparg; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c61b2e5036b480..a623078c7f7d52 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1620,7 +1620,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1881,7 +1881,7 @@ left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3202,7 +3202,7 @@ lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } @@ -3266,7 +3266,7 @@ case _SAVE_RETURN_OFFSET: { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - here); #endif #if TIER_TWO frame->return_offset = oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index de939bdcd6f3d7..b6904eafcedce6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2637,7 +2637,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2935,7 +2935,7 @@ STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -3076,7 +3076,7 @@ left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3626,7 +3626,7 @@ iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3640,7 +3640,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, frame->instr_ptr); + monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -3817,7 +3817,7 @@ } TARGET(FOR_ITER_GEN) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; @@ -3834,7 +3834,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -4214,7 +4214,7 @@ } #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } @@ -4249,7 +4249,7 @@ if (new_frame == NULL) { goto error; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -4355,7 +4355,7 @@ // _SAVE_RETURN_OFFSET { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - here); #endif #if TIER_TWO frame->return_offset = oparg; @@ -4433,7 +4433,7 @@ // _SAVE_RETURN_OFFSET { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - here); #endif #if TIER_TWO frame->return_offset = oparg; @@ -4503,7 +4503,7 @@ } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -4588,7 +4588,7 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; @@ -4602,7 +4602,7 @@ * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)&frame->instr_ptr[1]; + _PyCallCache *cache = (_PyCallCache *)&here[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -4631,7 +4631,7 @@ for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5127,6 +5127,7 @@ next_instr += 1; INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); + _Py_CODEUNIT *here = next_instr - 1; PyObject *kwnames; PyObject **args; PyObject *self_or_null; @@ -5172,7 +5173,7 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - here == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -5224,6 +5225,7 @@ next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *here = next_instr - 1; PyObject *kwargs = NULL; PyObject *callargs; PyObject *func; @@ -5287,7 +5289,7 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - here == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -5499,7 +5501,7 @@ lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = here; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } From 2b2819819a99fc03be20d9498864d6e98e805df1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Oct 2023 10:19:43 +0000 Subject: [PATCH 5/6] Change 'here' to 'this_instr' in bytecodes.c and generated cases --- Lib/test/test_generated_cases.py | 16 +- Python/bytecodes.c | 174 ++++++------ Python/executor_cases.c.h | 54 ++-- Python/generated_cases.c.h | 378 +++++++++++++------------- Tools/cases_generator/instructions.py | 6 +- Tools/cases_generator/stacking.py | 10 +- 6 files changed, 319 insertions(+), 319 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 6ce7876c2a19f7..475d7492040b7b 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -359,13 +359,13 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; - uint16_t counter = read_u16(&here[1].cache); - uint32_t extra = read_u32(&here[2].cache); + uint16_t counter = read_u16(&this_instr[1].cache); + uint32_t extra = read_u32(&this_instr[2].cache); STACK_SHRINK(1); DISPATCH(); } @@ -404,14 +404,14 @@ def test_macro_instruction(self): """ output = """ TARGET(OP1) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(OP1); PyObject *right; PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - uint16_t counter = read_u16(&here[1].cache); + uint16_t counter = read_u16(&this_instr[1].cache); op1(left, right); DISPATCH(); } @@ -421,7 +421,7 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP); PREDICTED(OP); - _Py_CODEUNIT *here = next_instr - 6; + _Py_CODEUNIT *this_instr = next_instr - 6; static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; @@ -431,13 +431,13 @@ def test_macro_instruction(self): right = stack_pointer[-1]; left = stack_pointer[-2]; { - uint16_t counter = read_u16(&here[1].cache); + uint16_t counter = read_u16(&this_instr[1].cache); op1(left, right); } // OP2 arg2 = stack_pointer[-3]; { - uint32_t extra = read_u32(&here[4].cache); + uint32_t extra = read_u32(&this_instr[4].cache); res = op2(arg2, left, right); } STACK_SHRINK(2); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 273d44559c3871..26d1d2197a3f28 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -80,7 +80,7 @@ dummy_func( pop_1_error: // Dummy locals. PyObject *dummy; - _Py_CODEUNIT *here; + _Py_CODEUNIT *this_instr; PyObject *attr; PyObject *attrs; PyObject *bottom; @@ -145,13 +145,13 @@ dummy_func( if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); - next_instr = here; + next_instr = this_instr; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - here->op.code = RESUME_CHECK; + this_instr->op.code = RESUME_CHECK; } } @@ -173,7 +173,7 @@ dummy_func( if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr = here; + next_instr = this_instr; } else { if (oparg < 2) { @@ -184,9 +184,9 @@ dummy_func( tstate, oparg > 0, frame, frame->instr_ptr); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); - if (frame->instr_ptr != here) { + if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = here; + next_instr = this_instr; DISPATCH(); } } @@ -313,13 +313,13 @@ dummy_func( inst(TO_BOOL, (unused/1, unused/2, value -- res)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); DECREF_INPUTS(); @@ -532,13 +532,13 @@ dummy_func( inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); DECREF_INPUTS(); @@ -679,13 +679,13 @@ dummy_func( inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -957,13 +957,13 @@ dummy_func( inst(SEND, (unused/1, receiver, v -- receiver, retval)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && @@ -1029,7 +1029,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, here, retval); + frame, this_instr, retval); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1089,7 +1089,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, here); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } @@ -1180,13 +1180,13 @@ dummy_func( inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -1242,14 +1242,14 @@ dummy_func( inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1363,14 +1363,14 @@ dummy_func( inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1688,7 +1688,7 @@ dummy_func( inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(here[1].cache); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -1702,20 +1702,20 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); #if ENABLE_SPECIALIZATION int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, here, global_super, arg); + frame, this_instr, global_super, arg); ERROR_IF(err, error); } @@ -1728,12 +1728,12 @@ dummy_func( if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, here, global_super, arg); + frame, this_instr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, here, global_super, arg); + frame, this_instr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -1811,14 +1811,14 @@ dummy_func( inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = here; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -2009,7 +2009,7 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2036,7 +2036,7 @@ dummy_func( STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2131,13 +2131,13 @@ dummy_func( inst(COMPARE_OP, (unused/1, left, right -- res)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -2268,20 +2268,20 @@ dummy_func( assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION - here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - if (here[1].cache > tstate->interp->optimizer_backedge_threshold && + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + if (this_instr[1].cache > tstate->interp->optimizer_backedge_threshold && // Double-check that the opcode isn't instrumented or something: - here->op.code == JUMP_BACKWARD) + this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); ERROR_IF(optimized < 0, error); if (optimized) { // Rewind and enter the executor: - assert(here->op.code == ENTER_EXECUTOR); - next_instr = here; + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; } - here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } #endif /* ENABLE_SPECIALIZATION */ } @@ -2318,7 +2318,7 @@ dummy_func( assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -2327,7 +2327,7 @@ dummy_func( assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -2443,13 +2443,13 @@ dummy_func( inst(FOR_ITER, (unused/1, iter -- iter, next)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2458,7 +2458,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2486,7 +2486,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2497,7 +2497,7 @@ dummy_func( /* Skip END_FOR */ target = next_instr + oparg + 1; } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); } op(_ITER_CHECK_LIST, (iter -- iter)) { @@ -2665,7 +2665,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -2900,9 +2900,9 @@ dummy_func( &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, here, function, arg); + frame, this_instr, function, arg); ERROR_IF(err, error); - INCREMENT_ADAPTIVE_COUNTER(here[1].cache); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); } @@ -2938,13 +2938,13 @@ dummy_func( total_args++; } #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; @@ -2974,7 +2974,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3128,7 +3128,7 @@ dummy_func( } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3176,7 +3176,7 @@ dummy_func( * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)&here[1]; + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL); DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; @@ -3205,7 +3205,7 @@ dummy_func( for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3575,7 +3575,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(next_instr - here == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -3672,7 +3672,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(next_instr - here == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -3793,13 +3793,13 @@ dummy_func( inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -3816,11 +3816,11 @@ dummy_func( inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, here); + tstate, frame, this_instr); ERROR_IF(next_opcode < 0, error); - next_instr = here; + next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(here[1].cache); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -3828,12 +3828,12 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { @@ -3842,9 +3842,9 @@ dummy_func( int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { @@ -3853,9 +3853,9 @@ dummy_func( int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { @@ -3870,9 +3870,9 @@ dummy_func( offset = 0; } #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { @@ -3887,9 +3887,9 @@ dummy_func( offset = oparg; } #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | !nflag; + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(EXTENDED_ARG, ( -- )) { @@ -3936,7 +3936,7 @@ dummy_func( op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - here); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a623078c7f7d52..8be60061415cdd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -119,13 +119,13 @@ PyObject *res; value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); @@ -379,13 +379,13 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); @@ -572,13 +572,13 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -897,13 +897,13 @@ PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -985,14 +985,14 @@ owner = stack_pointer[-1]; v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1128,14 +1128,14 @@ PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1618,14 +1618,14 @@ PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = here; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -1880,13 +1880,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -3201,13 +3201,13 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -3266,7 +3266,7 @@ case _SAVE_RETURN_OFFSET: { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - here); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b6904eafcedce6..5dc460028c193d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -15,7 +15,7 @@ next_instr += 1; INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *this_instr = next_instr - 1; static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); @@ -27,13 +27,13 @@ if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; - next_instr = here; + next_instr = this_instr; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - here->op.code = RESUME_CHECK; + this_instr->op.code = RESUME_CHECK; } DISPATCH(); } @@ -54,7 +54,7 @@ } TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; @@ -63,7 +63,7 @@ if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr = here; + next_instr = this_instr; } else { if (oparg < 2) { @@ -74,9 +74,9 @@ tstate, oparg > 0, frame, frame->instr_ptr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; - if (frame->instr_ptr != here) { + if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = here; + next_instr = this_instr; DISPATCH(); } } @@ -328,19 +328,19 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); - _Py_CODEUNIT *here = next_instr - 4; + _Py_CODEUNIT *this_instr = next_instr - 4; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); @@ -435,13 +435,13 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = read_u32(&here[2].cache); + uint32_t version = read_u32(&this_instr[2].cache); // This one is a bit weird, because we expect *some* failures: assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); @@ -703,7 +703,7 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; @@ -711,13 +711,13 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); @@ -957,7 +957,7 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; @@ -966,13 +966,13 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { next_instr = frame->instr_ptr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -1375,7 +1375,7 @@ next_instr += 2; INSTRUCTION_STATS(SEND); PREDICTED(SEND); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; @@ -1383,13 +1383,13 @@ v = stack_pointer[-1]; receiver = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && @@ -1456,7 +1456,7 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; @@ -1469,7 +1469,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, here, retval); + frame, this_instr, retval); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1525,7 +1525,7 @@ } TARGET(RERAISE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); PyObject *exc; @@ -1548,7 +1548,7 @@ assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, here); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } @@ -1687,18 +1687,18 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = frame->instr_ptr; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -1791,21 +1791,21 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); - _Py_CODEUNIT *here = next_instr - 5; + _Py_CODEUNIT *this_instr = next_instr - 5; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; owner = stack_pointer[-1]; v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1960,19 +1960,19 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *here = next_instr - 5; + _Py_CODEUNIT *this_instr = next_instr - 5; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -2017,14 +2017,14 @@ } TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { - uint16_t version = read_u16(&here[2].cache); + uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -2032,7 +2032,7 @@ } // _LOAD_GLOBAL_MODULE { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -2049,14 +2049,14 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { - uint16_t version = read_u16(&here[2].cache); + uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -2064,7 +2064,7 @@ } // _GUARD_BUILTINS_VERSION { - uint16_t version = read_u16(&here[3].cache); + uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -2072,7 +2072,7 @@ } // _LOAD_GLOBAL_BUILTINS { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -2478,12 +2478,12 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(here[1].cache); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -2492,7 +2492,7 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; @@ -2506,20 +2506,20 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); #if ENABLE_SPECIALIZATION int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, here, global_super, arg); + frame, this_instr, global_super, arg); if (err) goto pop_3_error; } @@ -2532,12 +2532,12 @@ if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, here, global_super, arg); + frame, this_instr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, here, global_super, arg); + frame, this_instr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -2628,21 +2628,21 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *here = next_instr - 10; + _Py_CODEUNIT *this_instr = next_instr - 10; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = here; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -2683,7 +2683,7 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); PyObject *owner; @@ -2692,7 +2692,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2706,7 +2706,7 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -2722,7 +2722,7 @@ } TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); PyObject *owner; @@ -2731,7 +2731,7 @@ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2739,7 +2739,7 @@ } // _LOAD_ATTR_MODULE { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); @@ -2758,7 +2758,7 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); PyObject *owner; @@ -2767,7 +2767,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2783,7 +2783,7 @@ } // _LOAD_ATTR_WITH_HINT { - uint16_t hint = read_u16(&here[4].cache); + uint16_t hint = read_u16(&this_instr[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); @@ -2811,7 +2811,7 @@ } TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); PyObject *owner; @@ -2820,14 +2820,14 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -2843,7 +2843,7 @@ } TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); PyObject *owner; @@ -2852,14 +2852,14 @@ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&here[6].cache); + PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2873,14 +2873,14 @@ } TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&here[2].cache); - uint32_t func_version = read_u32(&here[4].cache); - PyObject *fget = read_obj(&here[6].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); @@ -2900,20 +2900,20 @@ // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&here[2].cache); - uint32_t func_version = read_u32(&here[4].cache); - PyObject *getattribute = read_obj(&here[6].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2935,13 +2935,13 @@ STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); PyObject *owner; @@ -2949,7 +2949,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2963,7 +2963,7 @@ // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); PyDictValues *values = _PyDictOrValues_GetValues(dorv); @@ -2982,15 +2982,15 @@ } TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint32_t type_version = read_u32(&here[2].cache); - uint16_t hint = read_u16(&here[4].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -3034,7 +3034,7 @@ } TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); PyObject *owner; @@ -3042,7 +3042,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -3050,7 +3050,7 @@ // _STORE_ATTR_SLOT value = stack_pointer[-2]; { - uint16_t index = read_u16(&here[4].cache); + uint16_t index = read_u16(&this_instr[4].cache); char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -3067,7 +3067,7 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; @@ -3075,13 +3075,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -3316,27 +3316,27 @@ } TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD); CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION - here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - if (here[1].cache > tstate->interp->optimizer_backedge_threshold && + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + if (this_instr[1].cache > tstate->interp->optimizer_backedge_threshold && // Double-check that the opcode isn't instrumented or something: - here->op.code == JUMP_BACKWARD) + this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { // Rewind and enter the executor: - assert(here->op.code == ENTER_EXECUTOR); - next_instr = here; + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; } - here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); @@ -3364,7 +3364,7 @@ } TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; @@ -3372,7 +3372,7 @@ assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); STACK_SHRINK(1); @@ -3380,7 +3380,7 @@ } TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; @@ -3388,7 +3388,7 @@ assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); STACK_SHRINK(1); @@ -3396,7 +3396,7 @@ } TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyObject *value; @@ -3419,7 +3419,7 @@ assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -3428,7 +3428,7 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyObject *value; @@ -3451,7 +3451,7 @@ assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -3619,19 +3619,19 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3640,7 +3640,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -3659,7 +3659,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _Py_CODEUNIT *target; @@ -3674,7 +3674,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -3685,7 +3685,7 @@ /* Skip END_FOR */ target = next_instr + oparg + 1; } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3817,7 +3817,7 @@ } TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; @@ -3834,7 +3834,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -3992,7 +3992,7 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; @@ -4001,7 +4001,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -4014,14 +4014,14 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&here[4].cache); + uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { - PyObject *descr = read_obj(&here[6].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -4037,7 +4037,7 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; @@ -4046,14 +4046,14 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { - PyObject *descr = read_obj(&here[6].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -4069,7 +4069,7 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; @@ -4077,7 +4077,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -4090,14 +4090,14 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&here[4].cache); + uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - PyObject *descr = read_obj(&here[6].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -4109,7 +4109,7 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyObject *owner; @@ -4117,14 +4117,14 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { - PyObject *descr = read_obj(&here[6].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -4137,7 +4137,7 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); PyObject *owner; @@ -4146,7 +4146,7 @@ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&here[2].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -4161,7 +4161,7 @@ } // _LOAD_ATTR_METHOD_LAZY_DICT { - PyObject *descr = read_obj(&here[6].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -4176,7 +4176,7 @@ } TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); int is_meth = PEEK(oparg + 1) != NULL; @@ -4186,9 +4186,9 @@ &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, here, function, arg); + frame, this_instr, function, arg); if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(here[1].cache); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); } @@ -4197,7 +4197,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED(CALL); - _Py_CODEUNIT *here = next_instr - 4; + _Py_CODEUNIT *this_instr = next_instr - 4; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; @@ -4213,13 +4213,13 @@ total_args++; } #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; @@ -4249,7 +4249,7 @@ if (new_frame == NULL) { goto error; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -4289,7 +4289,7 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); PyObject *null; @@ -4323,7 +4323,7 @@ self_or_null = self; callable = func; { - uint32_t func_version = read_u32(&here[2].cache); + uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -4355,7 +4355,7 @@ // _SAVE_RETURN_OFFSET { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - here); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; @@ -4386,7 +4386,7 @@ } TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); PyObject *self_or_null; @@ -4401,7 +4401,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - uint32_t func_version = read_u32(&here[2].cache); + uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -4433,7 +4433,7 @@ // _SAVE_RETURN_OFFSET { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - here); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; @@ -4464,7 +4464,7 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); PyObject **args; @@ -4473,7 +4473,7 @@ args = stack_pointer - oparg; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&here[2].cache); + uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(tstate->interp->eval_frame, CALL); int argcount = oparg; if (self_or_null != NULL) { @@ -4503,7 +4503,7 @@ } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -4588,7 +4588,7 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; @@ -4602,7 +4602,7 @@ * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)&here[1]; + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -4631,7 +4631,7 @@ for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - here); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5127,7 +5127,7 @@ next_instr += 1; INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *this_instr = next_instr - 1; PyObject *kwnames; PyObject **args; PyObject *self_or_null; @@ -5173,7 +5173,7 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - here == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -5225,7 +5225,7 @@ next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *this_instr = next_instr - 1; PyObject *kwargs = NULL; PyObject *callargs; PyObject *func; @@ -5289,7 +5289,7 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - here == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -5492,7 +5492,7 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); - _Py_CODEUNIT *here = next_instr - 2; + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; @@ -5500,13 +5500,13 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(here[1].cache)) { - next_instr = here; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(here[1].cache); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -5535,15 +5535,15 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, here); + tstate, frame, this_instr); if (next_opcode < 0) goto error; - next_instr = here; + next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(here[1].cache); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -5551,24 +5551,24 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); PyObject *cond = POP(); @@ -5576,14 +5576,14 @@ int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); PyObject *cond = POP(); @@ -5591,14 +5591,14 @@ int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); PyObject *value = POP(); @@ -5612,14 +5612,14 @@ offset = 0; } #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); PyObject *value = POP(); @@ -5633,9 +5633,9 @@ offset = oparg; } #if ENABLE_SPECIALIZATION - here[1].cache = (here[1].cache << 1) | !nflag; + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 93e31c3d56e924..e9ed2a867ae2b3 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -61,7 +61,7 @@ class Instruction: # Computed by constructor always_exits: str # If the block always exits, its last line; else "" has_deopt: bool - needs_here: bool + needs_this_instr: bool cache_offset: int cache_effects: list[parsing.CacheEffect] input_effects: list[StackEffect] @@ -88,7 +88,7 @@ def __init__(self, inst: parsing.InstDef): effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) - self.needs_here = variable_used(self.inst, "here") or any(c.name != UNUSED for c in self.cache_effects) + self.needs_this_instr = variable_used(self.inst, "this_instr") or any(c.name != UNUSED for c in self.cache_effects) self.input_effects = [ effect for effect in inst.inputs if isinstance(effect, StackEffect) ] @@ -167,7 +167,7 @@ def write_body( if tier == TIER_ONE: out.emit( f"{typ}{ceffect.name} = " - f"{func}(&here[{active.offset + 1}].cache);" + f"{func}(&this_instr[{active.offset + 1}].cache);" ) else: out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index fea5bdcb82a26d..123e38c524f49d 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -365,17 +365,17 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: ] out.emit("") with out.block(f"TARGET({mac.name})"): - needs_here = any(part.instr.needs_here for part in parts) - if needs_here and not mac.predicted: - out.emit(f"_Py_CODEUNIT *here = frame->instr_ptr = next_instr;") + needs_this = any(part.instr.needs_this_instr for part in parts) + if needs_this and not mac.predicted: + out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;") else: out.emit(f"frame->instr_ptr = next_instr;") out.emit(f"next_instr += {mac.cache_offset+1};") out.emit(f"INSTRUCTION_STATS({mac.name});") if mac.predicted: out.emit(f"PREDICTED({mac.name});") - if needs_here: - out.emit(f"_Py_CODEUNIT *here = next_instr - {mac.cache_offset+1};") + if needs_this: + out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {mac.cache_offset+1};") out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) try: next_instr_is_set = write_components( From ad0616a15217a0a482e73546a8c6a99d532676ff Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Oct 2023 10:57:16 +0000 Subject: [PATCH 6/6] Use 'this_instr' more widely --- Python/bytecodes.c | 42 +++++++++++++------------- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 62 +++++++++++++++++++------------------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0486bc669129a9..d33ee3265d254d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -181,7 +181,7 @@ dummy_func( } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, frame->instr_ptr); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); if (frame->instr_ptr != this_instr) { @@ -267,7 +267,7 @@ dummy_func( * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -283,7 +283,7 @@ dummy_func( TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -657,7 +657,7 @@ dummy_func( STACK_SHRINK(2); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -680,7 +680,7 @@ dummy_func( inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -753,7 +753,7 @@ dummy_func( case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, frame->instr_ptr); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; @@ -807,7 +807,7 @@ dummy_func( inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, frame->instr_ptr, retval); + frame, this_instr, retval); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -831,7 +831,7 @@ dummy_func( PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, frame->instr_ptr, retval); + frame, this_instr, retval); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -977,7 +977,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -990,7 +990,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, frame->instr_ptr); + monitor_raise(tstate, frame, this_instr); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -1015,7 +1015,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1101,7 +1101,7 @@ dummy_func( else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, frame->instr_ptr); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } } @@ -1116,7 +1116,7 @@ dummy_func( } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, frame->instr_ptr); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } } @@ -2989,12 +2989,12 @@ dummy_func( if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3533,7 +3533,7 @@ dummy_func( : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, function, arg); + frame, this_instr, function, arg); ERROR_IF(err, error); GO_TO_INSTRUCTION(CALL_KW); } @@ -3590,12 +3590,12 @@ dummy_func( if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3638,18 +3638,18 @@ dummy_func( PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, func, arg); + frame, this_instr, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, func, arg); + frame, this_instr, func, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, func, arg); + frame, this_instr, func, arg); if (err < 0) { Py_CLEAR(result); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 8be60061415cdd..24de9eadf59048 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -573,7 +573,7 @@ v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 175169b1ddf4d7..40c8cd8798c93b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -71,7 +71,7 @@ } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, frame->instr_ptr); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->instr_ptr != this_instr) { @@ -237,7 +237,7 @@ } TARGET(INSTRUMENTED_END_FOR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyObject *value; @@ -249,7 +249,7 @@ * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -275,7 +275,7 @@ } TARGET(INSTRUMENTED_END_SEND) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyObject *value; @@ -285,7 +285,7 @@ TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, frame->instr_ptr)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -893,7 +893,7 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); PyObject *sub; @@ -919,7 +919,7 @@ STACK_SHRINK(2); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -967,7 +967,7 @@ v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = frame->instr_ptr; + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -1085,7 +1085,7 @@ } TARGET(RAISE_VARARGS) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; @@ -1101,7 +1101,7 @@ case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, frame->instr_ptr); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; @@ -1158,14 +1158,14 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, frame->instr_ptr, retval); + frame, this_instr, retval); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -1219,13 +1219,13 @@ } TARGET(INSTRUMENTED_RETURN_CONST) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, frame->instr_ptr, retval); + frame, this_instr, retval); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -1403,7 +1403,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1416,7 +1416,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, frame->instr_ptr); + monitor_raise(tstate, frame, this_instr); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -1432,7 +1432,7 @@ } TARGET(SEND_GEN) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); PyObject *v; @@ -1450,7 +1450,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1553,7 +1553,7 @@ } TARGET(END_ASYNC_FOR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); PyObject *exc; @@ -1568,7 +1568,7 @@ else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, frame->instr_ptr); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } STACK_SHRINK(2); @@ -1576,7 +1576,7 @@ } TARGET(CLEANUP_THROW) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); PyObject *exc_value; @@ -1598,7 +1598,7 @@ } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, frame->instr_ptr); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } STACK_SHRINK(1); @@ -4264,12 +4264,12 @@ if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -5107,7 +5107,7 @@ } TARGET(INSTRUMENTED_CALL_KW) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); int is_meth = PEEK(oparg + 2) != NULL; @@ -5117,7 +5117,7 @@ : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, function, arg); + frame, this_instr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } @@ -5188,12 +5188,12 @@ if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -5255,18 +5255,18 @@ PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, frame->instr_ptr, func, arg); + frame, this_instr, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, frame->instr_ptr, func, arg); + frame, this_instr, func, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, frame->instr_ptr, func, arg); + frame, this_instr, func, arg); if (err < 0) { Py_CLEAR(result); }