diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 16c1637e496033..d0060e169ad482 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -27,59 +27,65 @@ #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _GUARD_BOTH_INT 302 -#define _BINARY_OP_MULTIPLY_INT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_SUBTRACT_INT 305 -#define _GUARD_BOTH_FLOAT 306 -#define _BINARY_OP_MULTIPLY_FLOAT 307 -#define _BINARY_OP_ADD_FLOAT 308 -#define _BINARY_OP_SUBTRACT_FLOAT 309 -#define _GUARD_BOTH_UNICODE 310 -#define _BINARY_OP_ADD_UNICODE 311 -#define _BINARY_OP_INPLACE_ADD_UNICODE 312 -#define _POP_FRAME 313 -#define _GUARD_GLOBALS_VERSION 314 -#define _GUARD_BUILTINS_VERSION 315 -#define _LOAD_GLOBAL_MODULE 316 -#define _LOAD_GLOBAL_BUILTINS 317 -#define _GUARD_TYPE_VERSION 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _LOAD_ATTR_INSTANCE_VALUE 320 -#define _LOAD_ATTR_SLOT 321 -#define _GUARD_DORV_VALUES 322 -#define _STORE_ATTR_INSTANCE_VALUE 323 -#define _GUARD_TYPE_VERSION_STORE 324 -#define _STORE_ATTR_SLOT 325 -#define _IS_NONE 326 -#define _ITER_CHECK_LIST 327 -#define _ITER_JUMP_LIST 328 -#define _IS_ITER_EXHAUSTED_LIST 329 -#define _ITER_NEXT_LIST 330 -#define _ITER_CHECK_TUPLE 331 -#define _ITER_JUMP_TUPLE 332 -#define _IS_ITER_EXHAUSTED_TUPLE 333 -#define _ITER_NEXT_TUPLE 334 -#define _ITER_CHECK_RANGE 335 -#define _ITER_JUMP_RANGE 336 -#define _IS_ITER_EXHAUSTED_RANGE 337 -#define _ITER_NEXT_RANGE 338 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 339 -#define _GUARD_KEYS_VERSION 340 -#define _LOAD_ATTR_METHOD_WITH_VALUES 341 -#define _LOAD_ATTR_METHOD_NO_DICT 342 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 344 -#define _CHECK_PEP_523 345 -#define _CHECK_FUNCTION_EXACT_ARGS 346 -#define _CHECK_STACK_SPACE 347 -#define _INIT_CALL_PY_EXACT_ARGS 348 -#define _PUSH_FRAME 349 -#define _POP_JUMP_IF_FALSE 350 -#define _POP_JUMP_IF_TRUE 351 -#define _JUMP_TO_TOP 352 -#define _SAVE_CURRENT_IP 353 -#define _INSERT 354 +#define _END_FOR_MONITOR 302 +#define _END_SEND_MONITOR 303 +#define _GUARD_BOTH_INT 304 +#define _BINARY_OP_MULTIPLY_INT 305 +#define _BINARY_OP_ADD_INT 306 +#define _BINARY_OP_SUBTRACT_INT 307 +#define _GUARD_BOTH_FLOAT 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_ADD_FLOAT 310 +#define _BINARY_OP_SUBTRACT_FLOAT 311 +#define _GUARD_BOTH_UNICODE 312 +#define _BINARY_OP_ADD_UNICODE 313 +#define _BINARY_OP_INPLACE_ADD_UNICODE 314 +#define _POP_FRAME 315 +#define _MONITOR_RETURN 316 +#define _SUSPEND_GENERATOR 317 +#define _MONITOR_YIELD_VALUE 318 +#define _DO_YIELD 319 +#define _GUARD_GLOBALS_VERSION 320 +#define _GUARD_BUILTINS_VERSION 321 +#define _LOAD_GLOBAL_MODULE 322 +#define _LOAD_GLOBAL_BUILTINS 323 +#define _GUARD_TYPE_VERSION 324 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 325 +#define _LOAD_ATTR_INSTANCE_VALUE 326 +#define _LOAD_ATTR_SLOT 327 +#define _GUARD_DORV_VALUES 328 +#define _STORE_ATTR_INSTANCE_VALUE 329 +#define _GUARD_TYPE_VERSION_STORE 330 +#define _STORE_ATTR_SLOT 331 +#define _IS_NONE 332 +#define _ITER_CHECK_LIST 333 +#define _ITER_JUMP_LIST 334 +#define _IS_ITER_EXHAUSTED_LIST 335 +#define _ITER_NEXT_LIST 336 +#define _ITER_CHECK_TUPLE 337 +#define _ITER_JUMP_TUPLE 338 +#define _IS_ITER_EXHAUSTED_TUPLE 339 +#define _ITER_NEXT_TUPLE 340 +#define _ITER_CHECK_RANGE 341 +#define _ITER_JUMP_RANGE 342 +#define _IS_ITER_EXHAUSTED_RANGE 343 +#define _ITER_NEXT_RANGE 344 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 345 +#define _GUARD_KEYS_VERSION 346 +#define _LOAD_ATTR_METHOD_WITH_VALUES 347 +#define _LOAD_ATTR_METHOD_NO_DICT 348 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 349 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 350 +#define _CHECK_PEP_523 351 +#define _CHECK_FUNCTION_EXACT_ARGS 352 +#define _CHECK_STACK_SPACE 353 +#define _INIT_CALL_PY_EXACT_ARGS 354 +#define _PUSH_FRAME 355 +#define _POP_JUMP_IF_FALSE 356 +#define _POP_JUMP_IF_TRUE 357 +#define _JUMP_TO_TOP 358 +#define _SAVE_CURRENT_IP 359 +#define _INSERT 360 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -119,10 +125,14 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case END_FOR: return 2; + case _END_FOR_MONITOR: + return 2; case INSTRUMENTED_END_FOR: return 2; case END_SEND: return 2; + case _END_SEND_MONITOR: + return 2; case INSTRUMENTED_END_SEND: return 2; case UNARY_NEGATIVE: @@ -223,6 +233,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case RETURN_VALUE: return 1; + case _MONITOR_RETURN: + return 1; case INSTRUMENTED_RETURN_VALUE: return 1; case RETURN_CONST: @@ -239,10 +251,16 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case SEND_GEN: return 2; - case INSTRUMENTED_YIELD_VALUE: + case _SUSPEND_GENERATOR: + return 1; + case _MONITOR_YIELD_VALUE: + return 1; + case _DO_YIELD: return 1; case YIELD_VALUE: return 1; + case INSTRUMENTED_YIELD_VALUE: + return 1; case POP_EXCEPT: return 1; case RERAISE: @@ -677,10 +695,14 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case END_FOR: return 0; + case _END_FOR_MONITOR: + return 2; case INSTRUMENTED_END_FOR: return 0; case END_SEND: return 1; + case _END_SEND_MONITOR: + return 2; case INSTRUMENTED_END_SEND: return 1; case UNARY_NEGATIVE: @@ -781,6 +803,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RETURN_VALUE: return 0; + case _MONITOR_RETURN: + return 1; case INSTRUMENTED_RETURN_VALUE: return 0; case RETURN_CONST: @@ -797,10 +821,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case SEND_GEN: return 2; - case INSTRUMENTED_YIELD_VALUE: + case _SUSPEND_GENERATOR: + return 1; + case _MONITOR_YIELD_VALUE: + return 1; + case _DO_YIELD: return 1; case YIELD_VALUE: return 1; + case INSTRUMENTED_YIELD_VALUE: + return 1; case POP_EXCEPT: return 0; case RERAISE: @@ -1281,8 +1311,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [END_FOR] = { true, INSTR_FMT_IX, 0 }, + [_END_FOR_MONITOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, 0 }, + [_END_SEND_MONITOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, @@ -1333,6 +1365,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, 0 }, [_POP_FRAME] = { true, INSTR_FMT_IX, 0 }, [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, + [_MONITOR_RETURN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG }, @@ -1341,8 +1374,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_SUSPEND_GENERATOR] = { true, INSTR_FMT_IX, 0 }, + [_MONITOR_YIELD_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [_DO_YIELD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, 0 }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, @@ -1692,6 +1728,8 @@ extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_EXIT_TRACE] = "_EXIT_TRACE", [_SET_IP] = "_SET_IP", + [_END_FOR_MONITOR] = "_END_FOR_MONITOR", + [_END_SEND_MONITOR] = "_END_SEND_MONITOR", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", @@ -1704,6 +1742,10 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_POP_FRAME] = "_POP_FRAME", + [_MONITOR_RETURN] = "_MONITOR_RETURN", + [_SUSPEND_GENERATOR] = "_SUSPEND_GENERATOR", + [_MONITOR_YIELD_VALUE] = "_MONITOR_YIELD_VALUE", + [_DO_YIELD] = "_DO_YIELD", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-26-03.gh-issue-104909.OWEgB8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-26-03.gh-issue-104909.OWEgB8.rst new file mode 100644 index 00000000000000..49e3a8305e2124 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-26-03.gh-issue-104909.OWEgB8.rst @@ -0,0 +1,2 @@ +Implement instrumented instructions in terms of micro-ops and make it +explicit that instrumented instructions are tier 1 only. diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 61b1db9e5a1543..ec85a1ae105fff 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -216,6 +216,10 @@ break; } + case _SUSPEND_GENERATOR: { + break; + } + case POP_EXCEPT: { STACK_SHRINK(1); break; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0f89779fb9245f..a1375341ec6c19 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -163,14 +163,13 @@ dummy_func( } inst(INSTRUMENTED_RESUME, (--)) { + TIER_ONE_ONLY; /* Possible performance enhancement: * We need to check the eval breaker anyway, can we * combine the instrument verison check and the eval breaker test? */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - goto error; - } + ERROR_IF(_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp), error); next_instr--; } else { @@ -259,34 +258,34 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + op(_END_FOR_MONITOR, (receiver, value -- receiver, value)) { + TIER_ONE_ONLY; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; - } + ERROR_IF(monitor_stop_iteration(tstate, frame, next_instr-1), error); PyErr_SetRaisedException(NULL); } - DECREF_INPUTS(); } + macro(INSTRUMENTED_END_FOR) = _END_FOR_MONITOR + POP_TOP + POP_TOP; + inst(END_SEND, (receiver, value -- value)) { Py_DECREF(receiver); } - inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + op(_END_SEND_MONITOR, (receiver, value -- receiver, value)) { + TIER_ONE_ONLY; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; - } + ERROR_IF(monitor_stop_iteration(tstate, frame, next_instr-1), error); PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); } + macro(INSTRUMENTED_END_SEND) = _END_SEND_MONITOR + END_SEND; + inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); DECREF_INPUTS(); @@ -741,6 +740,7 @@ dummy_func( } inst(RAISE_VARARGS, (args[oparg] -- )) { + TIER_ONE_ONLY; PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -806,50 +806,32 @@ dummy_func( _SAVE_CURRENT_IP + // Sets frame->prev_instr _POP_FRAME; - inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + op(_MONITOR_RETURN, (retval -- retval)) { + TIER_ONE_ONLY; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - if (err) goto error; - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + ERROR_IF(err, error); } + macro(INSTRUMENTED_RETURN_VALUE) = + _MONITOR_RETURN + + _SET_IP + // Tier 2 only; special-cased oparg + _SAVE_CURRENT_IP + // Sets frame->prev_instr + _POP_FRAME; + macro(RETURN_CONST) = LOAD_CONST + _SET_IP + // Tier 2 only; special-cased oparg _SAVE_CURRENT_IP + // Sets frame->prev_instr _POP_FRAME; - inst(INSTRUMENTED_RETURN_CONST, (--)) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } + macro(INSTRUMENTED_RETURN_CONST) = + LOAD_CONST + + _MONITOR_RETURN + + _SET_IP + // Tier 2 only; special-cased oparg + _SAVE_CURRENT_IP + // Sets frame->prev_instr + _POP_FRAME; inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; @@ -960,6 +942,7 @@ dummy_func( }; inst(SEND, (unused/1, receiver, v -- receiver, retval)) { + TIER_ONE_ONLY; #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1026,35 +1009,29 @@ dummy_func( DISPATCH_INLINED(gen_frame); } - inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { + op(_SUSPEND_GENERATOR, (val -- val)) { +#if TIER_ONE assert(frame != &entry_frame); - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ +#endif PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); + } + + op(_MONITOR_YIELD_VALUE, (val -- val)) { + TIER_ONE_ONLY; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); - if (err) goto error; - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + frame, next_instr-1, val); + ERROR_IF(err, error); } - inst(YIELD_VALUE, (retval -- unused)) { + op(_DO_YIELD, (retval -- unused)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1065,12 +1042,22 @@ dummy_func( goto resume_frame; } + macro(YIELD_VALUE) = + _SUSPEND_GENERATOR + + _DO_YIELD; + + macro(INSTRUMENTED_YIELD_VALUE) = + _SUSPEND_GENERATOR + + _MONITOR_YIELD_VALUE + + _DO_YIELD; + inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); } inst(RERAISE, (values[oparg], exc -- values[oparg])) { + TIER_ONE_ONLY; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1092,6 +1079,7 @@ dummy_func( } inst(END_ASYNC_FOR, (awaitable, exc -- )) { + TIER_ONE_ONLY; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { DECREF_INPUTS(); @@ -1105,6 +1093,7 @@ dummy_func( } inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { + TIER_ONE_ONLY; assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1132,7 +1121,6 @@ dummy_func( } } - inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -1687,6 +1675,7 @@ dummy_func( } inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) { + TIER_ONE_ONLY; _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions @@ -2431,6 +2420,7 @@ dummy_func( }; inst(FOR_ITER, (unused/1, iter -- iter, next)) { + TIER_ONE_ONLY; #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2465,6 +2455,7 @@ dummy_func( } inst(INSTRUMENTED_FOR_ITER, ( -- )) { + TIER_ONE_ONLY; _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -2886,6 +2877,7 @@ dummy_func( } inst(INSTRUMENTED_CALL, ( -- )) { + TIER_ONE_ONLY; int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -3611,6 +3603,7 @@ dummy_func( } inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { + TIER_ONE_ONLY; GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } @@ -3812,6 +3805,7 @@ dummy_func( } inst(INSTRUMENTED_INSTRUCTION, ( -- )) { + TIER_ONE_ONLY; int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); ERROR_IF(next_opcode < 0, error); @@ -3826,15 +3820,18 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { + TIER_ONE_ONLY; INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { + TIER_ONE_ONLY; CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { + TIER_ONE_ONLY; PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; @@ -3847,6 +3844,7 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { + TIER_ONE_ONLY; PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; @@ -3859,6 +3857,7 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { + TIER_ONE_ONLY; PyObject *value = POP(); _Py_CODEUNIT *here = next_instr - 1; int flag = Py_IsNone(value); @@ -3877,6 +3876,7 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { + TIER_ONE_ONLY; PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 981db6973f281a..bc2096cd86261f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -825,6 +825,18 @@ break; } + case _SUSPEND_GENERATOR: { + PyObject *val; + val = stack_pointer[-1]; +#if TIER_ONE + assert(frame != &entry_frame); +#endif + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + break; + } + case POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 17df44019a6581..adb28420b5c251 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -39,14 +39,13 @@ } TARGET(INSTRUMENTED_RESUME) { + TIER_ONE_ONLY; /* Possible performance enhancement: * We need to check the eval breaker anyway, can we * combine the instrument verison check and the eval breaker test? */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - goto error; - } + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) goto error; next_instr--; } else { @@ -190,19 +189,28 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value; PyObject *receiver; + // _END_FOR_MONITOR value = stack_pointer[-1]; receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; + { + TIER_ONE_ONLY; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) goto error; + PyErr_SetRaisedException(NULL); } - PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); - Py_DECREF(value); + // POP_TOP + { + Py_DECREF(value); + } + // POP_TOP + value = receiver; + { + Py_DECREF(value); + } STACK_SHRINK(2); DISPATCH(); } @@ -221,16 +229,21 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value; PyObject *receiver; + // _END_SEND_MONITOR value = stack_pointer[-1]; receiver = stack_pointer[-2]; - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; + { + TIER_ONE_ONLY; + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) goto error; + PyErr_SetRaisedException(NULL); } - PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); + // END_SEND + { + Py_DECREF(receiver); + } STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -947,6 +960,7 @@ TARGET(RAISE_VARARGS) { PyObject **args; args = stack_pointer - oparg; + TIER_ONE_ONLY; PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -1024,23 +1038,50 @@ TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval; + // _MONITOR_RETURN retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; + { + TIER_ONE_ONLY; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; + } + // _SAVE_CURRENT_IP + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding _SET_IP + frame->prev_instr--; + #endif + } + // _POP_FRAME STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + { + assert(EMPTY()); + #if TIER_ONE + assert(frame != &entry_frame); + #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + DISPATCH(); } TARGET(RETURN_CONST) { @@ -1089,23 +1130,56 @@ } TARGET(INSTRUMENTED_RETURN_CONST) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + PyObject *value; + PyObject *retval; + // LOAD_CONST + { + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + } + // _MONITOR_RETURN + retval = value; + { + TIER_ONE_ONLY; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; + } + // _SAVE_CURRENT_IP + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding _SET_IP + frame->prev_instr--; + #endif + } + // _POP_FRAME + { + assert(EMPTY()); + #if TIER_ONE + assert(frame != &entry_frame); + #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + DISPATCH(); } TARGET(GET_AITER) { @@ -1236,6 +1310,7 @@ PyObject *retval; v = stack_pointer[-1]; receiver = stack_pointer[-2]; + TIER_ONE_ONLY; #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1309,47 +1384,76 @@ DISPATCH_INLINED(gen_frame); } - TARGET(INSTRUMENTED_YIELD_VALUE) { + TARGET(YIELD_VALUE) { + PyObject *val; PyObject *retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); - if (err) goto error; - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + // _SUSPEND_GENERATOR + val = stack_pointer[-1]; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + } + // _DO_YIELD + retval = val; + { + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + PyGenObject *gen = _PyFrame_GetGenerator(frame); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } } - TARGET(YIELD_VALUE) { + TARGET(INSTRUMENTED_YIELD_VALUE) { + PyObject *val; PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - assert(frame != &entry_frame); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + // _SUSPEND_GENERATOR + val = stack_pointer[-1]; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + } + // _MONITOR_YIELD_VALUE + { + TIER_ONE_ONLY; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, val); + if (err) goto error; + } + // _DO_YIELD + retval = val; + { + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + PyGenObject *gen = _PyFrame_GetGenerator(frame); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } } TARGET(POP_EXCEPT) { @@ -1366,6 +1470,7 @@ PyObject **values; exc = stack_pointer[-1]; values = stack_pointer - 1 - oparg; + TIER_ONE_ONLY; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1391,6 +1496,7 @@ PyObject *awaitable; exc = stack_pointer[-1]; awaitable = stack_pointer[-2]; + TIER_ONE_ONLY; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -1415,6 +1521,7 @@ exc_value = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; sub_iter = stack_pointer[-3]; + TIER_ONE_ONLY; assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -2200,6 +2307,7 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + TIER_ONE_ONLY; _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions @@ -3226,6 +3334,7 @@ PyObject *iter; PyObject *next; iter = stack_pointer[-1]; + TIER_ONE_ONLY; #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3264,6 +3373,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { + TIER_ONE_ONLY; _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3730,6 +3840,7 @@ } TARGET(INSTRUMENTED_CALL) { + TIER_ONE_ONLY; int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -4720,6 +4831,7 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + TIER_ONE_ONLY; GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } @@ -5002,6 +5114,7 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { + TIER_ONE_ONLY; int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -5016,17 +5129,20 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { + TIER_ONE_ONLY; INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { + TIER_ONE_ONLY; CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + TIER_ONE_ONLY; PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; @@ -5041,6 +5157,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + TIER_ONE_ONLY; PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; @@ -5055,6 +5172,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + TIER_ONE_ONLY; PyObject *value = POP(); _Py_CODEUNIT *here = next_instr - 1; int flag = Py_IsNone(value); @@ -5075,6 +5193,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + TIER_ONE_ONLY; PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset;