From 830084cf59e4e422424663a78677b9c1f4940197 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:28:34 -0700 Subject: [PATCH 01/17] Specialize CALL_FUNCTION_EX --- Include/internal/pycore_call.h | 10 ++ Include/internal/pycore_code.h | 1 + Include/internal/pycore_opcode.h | 67 +++++------ Include/opcode.h | 86 +++++++------- Lib/importlib/_bootstrap_external.py | 4 +- Lib/opcode.py | 11 ++ Objects/call.c | 20 ++-- Python/ceval.c | 160 +++++++++++++++++++++++++++ Python/opcode_targets.h | 58 +++++----- Python/specialize.c | 74 +++++++++++++ 10 files changed, 375 insertions(+), 116 deletions(-) diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 55378e3dfebf24..5d9342b562b002 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -116,6 +116,16 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); } +PyObject *const * +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames); + +void +_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, + PyObject *kwnames); + +void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames); #ifdef __cplusplus } diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bf5945435c1774..29caf4d524c145 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -244,6 +244,7 @@ extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_C extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames); +extern int _Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 15925511cc1f41..88feb7d693914b 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -50,6 +50,7 @@ const uint8_t _PyOpcode_Caches[256] = { [COMPARE_OP] = 2, [LOAD_GLOBAL] = 5, [BINARY_OP] = 1, + [CALL_FUNCTION_EX] = 4, [CALL] = 4, }; @@ -88,6 +89,10 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_ADAPTIVE] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_PY_KWARGS] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_PY_NO_KWARGS] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, @@ -297,36 +302,36 @@ static const char *const _PyOpcode_OpName[263] = { [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [CALL_FUNCTION_EX_ADAPTIVE] = "CALL_FUNCTION_EX_ADAPTIVE", + [CALL_FUNCTION_EX_PY_KWARGS] = "CALL_FUNCTION_EX_PY_KWARGS", + [CALL_FUNCTION_EX_PY_NO_KWARGS] = "CALL_FUNCTION_EX_PY_NO_KWARGS", + [STORE_SUBSCR] = "STORE_SUBSCR", + [DELETE_SUBSCR] = "DELETE_SUBSCR", + [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", [COMPARE_OP_ADAPTIVE] = "COMPARE_OP_ADAPTIVE", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", - [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", + [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -353,7 +358,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -361,7 +366,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -381,9 +386,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -393,26 +398,30 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [RESUME_QUICK] = "RESUME_QUICK", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [RESUME_QUICK] = "RESUME_QUICK", [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", + [CALL] = "CALL", + [KW_NAMES] = "KW_NAMES", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", @@ -421,10 +430,6 @@ static const char *const _PyOpcode_OpName[263] = { [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [181] = "<181>", - [182] = "<182>", - [183] = "<183>", - [184] = "<184>", [185] = "<185>", [186] = "<186>", [187] = "<187>", @@ -507,10 +512,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 181: \ - case 182: \ - case 183: \ - case 184: \ case 185: \ case 186: \ case 187: \ diff --git a/Include/opcode.h b/Include/opcode.h index 42825df6217b46..50958474c5acef 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -158,47 +158,51 @@ extern "C" { #define CALL_NO_KW_STR_1 47 #define CALL_NO_KW_TUPLE_1 48 #define CALL_NO_KW_TYPE_1 56 -#define COMPARE_OP_ADAPTIVE 57 -#define COMPARE_OP_FLOAT_JUMP 58 -#define COMPARE_OP_INT_JUMP 59 -#define COMPARE_OP_STR_JUMP 62 -#define EXTENDED_ARG_QUICK 63 -#define FOR_ITER_ADAPTIVE 64 -#define FOR_ITER_LIST 65 -#define FOR_ITER_RANGE 66 -#define JUMP_BACKWARD_QUICK 67 -#define LOAD_ATTR_ADAPTIVE 72 -#define LOAD_ATTR_CLASS 73 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 -#define LOAD_ATTR_INSTANCE_VALUE 77 -#define LOAD_ATTR_MODULE 78 -#define LOAD_ATTR_PROPERTY 79 -#define LOAD_ATTR_SLOT 80 -#define LOAD_ATTR_WITH_HINT 81 -#define LOAD_ATTR_METHOD_LAZY_DICT 86 -#define LOAD_ATTR_METHOD_NO_DICT 113 -#define LOAD_ATTR_METHOD_WITH_DICT 121 -#define LOAD_ATTR_METHOD_WITH_VALUES 141 -#define LOAD_CONST__LOAD_FAST 143 -#define LOAD_FAST__LOAD_CONST 153 -#define LOAD_FAST__LOAD_FAST 154 -#define LOAD_GLOBAL_ADAPTIVE 158 -#define LOAD_GLOBAL_BUILTIN 159 -#define LOAD_GLOBAL_MODULE 160 -#define RESUME_QUICK 161 -#define STORE_ATTR_ADAPTIVE 166 -#define STORE_ATTR_INSTANCE_VALUE 167 -#define STORE_ATTR_SLOT 168 -#define STORE_ATTR_WITH_HINT 169 -#define STORE_FAST__LOAD_FAST 170 -#define STORE_FAST__STORE_FAST 173 -#define STORE_SUBSCR_ADAPTIVE 174 -#define STORE_SUBSCR_DICT 175 -#define STORE_SUBSCR_LIST_INT 176 -#define UNPACK_SEQUENCE_ADAPTIVE 177 -#define UNPACK_SEQUENCE_LIST 178 -#define UNPACK_SEQUENCE_TUPLE 179 -#define UNPACK_SEQUENCE_TWO_TUPLE 180 +#define CALL_FUNCTION_EX_ADAPTIVE 57 +#define CALL_FUNCTION_EX_PY_KWARGS 58 +#define CALL_FUNCTION_EX_PY_NO_KWARGS 59 +#define CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS 62 +#define COMPARE_OP_ADAPTIVE 63 +#define COMPARE_OP_FLOAT_JUMP 64 +#define COMPARE_OP_INT_JUMP 65 +#define COMPARE_OP_STR_JUMP 66 +#define EXTENDED_ARG_QUICK 67 +#define FOR_ITER_ADAPTIVE 72 +#define FOR_ITER_LIST 73 +#define FOR_ITER_RANGE 76 +#define JUMP_BACKWARD_QUICK 77 +#define LOAD_ATTR_ADAPTIVE 78 +#define LOAD_ATTR_CLASS 79 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 80 +#define LOAD_ATTR_INSTANCE_VALUE 81 +#define LOAD_ATTR_MODULE 86 +#define LOAD_ATTR_PROPERTY 113 +#define LOAD_ATTR_SLOT 121 +#define LOAD_ATTR_WITH_HINT 141 +#define LOAD_ATTR_METHOD_LAZY_DICT 143 +#define LOAD_ATTR_METHOD_NO_DICT 153 +#define LOAD_ATTR_METHOD_WITH_DICT 154 +#define LOAD_ATTR_METHOD_WITH_VALUES 158 +#define LOAD_CONST__LOAD_FAST 159 +#define LOAD_FAST__LOAD_CONST 160 +#define LOAD_FAST__LOAD_FAST 161 +#define LOAD_GLOBAL_ADAPTIVE 166 +#define LOAD_GLOBAL_BUILTIN 167 +#define LOAD_GLOBAL_MODULE 168 +#define RESUME_QUICK 169 +#define STORE_ATTR_ADAPTIVE 170 +#define STORE_ATTR_INSTANCE_VALUE 173 +#define STORE_ATTR_SLOT 174 +#define STORE_ATTR_WITH_HINT 175 +#define STORE_FAST__LOAD_FAST 176 +#define STORE_FAST__STORE_FAST 177 +#define STORE_SUBSCR_ADAPTIVE 178 +#define STORE_SUBSCR_DICT 179 +#define STORE_SUBSCR_LIST_INT 180 +#define UNPACK_SEQUENCE_ADAPTIVE 181 +#define UNPACK_SEQUENCE_LIST 182 +#define UNPACK_SEQUENCE_TUPLE 183 +#define UNPACK_SEQUENCE_TWO_TUPLE 184 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b3c31b9659d849..4db6f3faf7857d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -413,6 +413,8 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3507 (Set lineno of module's RESUME to 0) # Python 3.12a1 3508 (Add CLEANUP_THROW) # Python 3.12a1 3509 (Conditional jumps only jump forward) +# Python 3.12a1 3510 (Specialization/Cache for CALL_FUNCTION_EX) + # Python 3.13 will start with 3550 @@ -425,7 +427,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3509).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3510).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 690923061418bd..bd48c594851084 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -311,6 +311,12 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TUPLE_1", "CALL_NO_KW_TYPE_1", ], + "CALL_FUNCTION_EX": [ + "CALL_FUNCTION_EX_ADAPTIVE", + "CALL_FUNCTION_EX_PY_KWARGS", + "CALL_FUNCTION_EX_PY_NO_KWARGS", + "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", + ], "COMPARE_OP": [ "COMPARE_OP_ADAPTIVE", "COMPARE_OP_FLOAT_JUMP", @@ -434,6 +440,11 @@ def pseudo_op(name, op, real_ops): "func_version": 2, "min_args": 1, }, + "CALL_FUNCTION_EX": { + "counter": 1, + "func_version": 2, + "min_args": 1, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Objects/call.c b/Objects/call.c index c2509db2a9a263..96a5b07d646a9f 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -7,16 +7,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() -static PyObject *const * -_PyStack_UnpackDict(PyThreadState *tstate, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject **p_kwnames); - -static void -_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, - PyObject *kwnames); - - static PyObject * null_error(PyThreadState *tstate) { @@ -964,7 +954,7 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames) The newly allocated argument vector supports PY_VECTORCALL_ARGUMENTS_OFFSET. When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */ -static PyObject *const * +PyObject *const * _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames) @@ -1036,7 +1026,7 @@ _PyStack_UnpackDict(PyThreadState *tstate, return stack; } -static void +void _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -1044,6 +1034,12 @@ _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, for (Py_ssize_t i = 0; i < n; i++) { Py_DECREF(stack[i]); } + _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames); +} + +void +_PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames) +{ PyMem_Free((PyObject **)stack - 1); Py_DECREF(kwnames); } diff --git a/Python/ceval.c b/Python/ceval.c index 82b5422c188ea7..b0102b7cd4a7c9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -25,6 +25,7 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_call.h" // _PyStack_UnpackDict/_Free #include "pycore_dict.h" #include "dictobject.h" @@ -4751,6 +4752,165 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (result == NULL) { goto error; } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX_ADAPTIVE) { + _PyCallCache *cache = (_PyCallCache *)next_instr; + PyObject *func, *callargs, *kwargs = NULL; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + if (oparg & 0x01) { + kwargs = TOP(); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(PyDict_CheckExact(kwargs)); + callargs = SECOND(); + func = THIRD(); + } + else { + callargs = TOP(); + func = SECOND(); + } + + int err = _Py_Specialize_CallEx(func, next_instr, callargs, kwargs); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(CALL_FUNCTION_EX, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + } + + TARGET(CALL_FUNCTION_EX_PY_KWARGS) { + assert(oparg & 0x01); + PyObject *func, *callargs, *kwargs = NULL; + kwargs = TOP(); + callargs = SECOND(); + func = THIRD(); + DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + DEOPT_IF(kwargs == NULL || PyDict_GET_SIZE(kwargs) <= 0, CALL_FUNCTION_EX); + DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); + DEOPT_IF(((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); + STAT_INC(CALL_FUNCTION_EX, hit); + STACK_SHRINK(4); /* Get rid of kwargs, callargs, func and NULL */ + + PyObject *kwnames; + PyObject *const *newargs = _PyStack_UnpackDict(tstate, + _PyTuple_ITEMS(callargs), nargs, + kwargs, &kwnames); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + CALL_STAT_INC(frames_pushed); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + Py_DECREF(kwargs); + Py_DECREF(callargs); + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + + TARGET(CALL_FUNCTION_EX_PY_NO_KWARGS) { + assert(!(oparg & 0x01)); + PyObject *func, *callargs; + callargs = TOP(); + func = SECOND(); + DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); + STACK_SHRINK(3); + STAT_INC(CALL_FUNCTION_EX, hit); + + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + CALL_STAT_INC(frames_pushed); + + for (Py_ssize_t i = 0; i < nargs; ++i) { + /* We need to incref all our args since the new frame steals the references. */ + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + &PyTuple_GET_ITEM(callargs, 0), nargs, NULL + ); + Py_DECREF(callargs); + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + + TARGET(CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS) { + PyObject *func, *callargs, *kwargs = NULL; + int has_kwargs = oparg & 0x01; + if (has_kwargs) { + kwargs = TOP(); + callargs = SECOND(); + func = THIRD(); + } + else { + callargs = TOP(); + func = SECOND(); + } + DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + DEOPT_IF(!PyCFunction_CheckExact(func), CALL_FUNCTION_EX); + DEOPT_IF(PyCFunction_GET_FLAGS(func) != (METH_VARARGS | METH_KEYWORDS), CALL); + DEOPT_IF(kwargs != NULL && + ((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); + STAT_INC(CALL_FUNCTION_EX, hit); + STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ + + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + + /* res = func(self, args, nargs, kwnames) */ + PyCFunction cfunc = PyCFunction_GET_FUNCTION(func); + PyObject *res = ((PyCFunctionWithKeywords)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(func), + callargs, + (kwargs == NULL || PyDict_GET_SIZE(kwargs) == 0) ? NULL : kwargs + ); + + Py_XDECREF(kwargs); + Py_DECREF(callargs); + _Py_LeaveRecursiveCallTstate(tstate); + assert(TOP() == NULL); + SET_TOP(res); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c1ff367d4fd38d..32cbf25a9929bd 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -56,36 +56,36 @@ static void *opcode_targets[256] = { &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_CALL_FUNCTION_EX_ADAPTIVE, + &&TARGET_CALL_FUNCTION_EX_PY_KWARGS, + &&TARGET_CALL_FUNCTION_EX_PY_NO_KWARGS, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, + &&TARGET_CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_INT_JUMP, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_EXTENDED_ARG_QUICK, - &&TARGET_FOR_ITER_ADAPTIVE, - &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_RANGE, - &&TARGET_JUMP_BACKWARD_QUICK, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_FOR_ITER_ADAPTIVE, + &&TARGET_FOR_ITER_LIST, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_FOR_ITER_RANGE, + &&TARGET_JUMP_BACKWARD_QUICK, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,26 +152,30 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_RESUME_QUICK, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_LOAD_GLOBAL_ADAPTIVE, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_RESUME_QUICK, &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_CALL, + &&TARGET_KW_NAMES, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_CALL, - &&TARGET_KW_NAMES, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, @@ -250,9 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index b7c321e4878b98..d012ba4b119df1 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -24,6 +24,7 @@ uint8_t _PyOpcode_Adaptive[256] = { [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, [CALL] = CALL_ADAPTIVE, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, [BINARY_OP] = BINARY_OP_ADAPTIVE, [COMPARE_OP] = COMPARE_OP_ADAPTIVE, @@ -1800,6 +1801,79 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return 0; } + +static int +specialize_py_callex(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs, int oparg) +{ + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0) { + assert(oparg & 0x01); + if (((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + /* kwargs keys should be unicode */ + SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); + return 1; + } + _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_KWARGS); + return 0; + } + else { + if (oparg & 0x01) { + /* Bad TOS dict or something */ + SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); + return 1; + } + _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_NO_KWARGS); + return 0; + } +} + + +int +_Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs) +{ + int oparg = _Py_OPARG(*instr); + assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL); + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + int fail; + if (!PyTuple_CheckExact(args)) { + fail = 1; + goto fail_please; + } + if (Py_IS_TYPE(callable, &PyFunction_Type)) { + fail = specialize_py_callex(callable, instr, args, kwargs, oparg); + } + else if (PyCFunction_CheckExact(callable)) { + PyCFunctionObject *func = (PyCFunctionObject *)callable; + switch (func->m_ml->ml_flags & + (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_VARARGS | METH_KEYWORDS: + _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS); + break; + default: + fail = 1; + } + } + else { + SPECIALIZATION_FAIL(CALL_FUNCTION_EX, call_fail_kind(callable)); + fail = 1; + } + +fail_please: + if (fail) { + STAT_INC(CALL_FUNCTION_EX, failure); + assert(!PyErr_Occurred()); + cache->counter = adaptive_counter_backoff(cache->counter); + } + else { + STAT_INC(CALL_FUNCTION_EX, success); + assert(!PyErr_Occurred()); + cache->counter = miss_counter_start(); + } + return 0; +} + + #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) From d9fbb11db91157cbc9c02823a5d1cfcd0454dde0 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 23:32:12 +0000 Subject: [PATCH 02/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst new file mode 100644 index 00000000000000..a6dc08f6092ba3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -0,0 +1 @@ +Specialize complex calls. From e7246663df1b3bfa32e92791b8c081aada995258 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:50:13 -0700 Subject: [PATCH 03/17] fix refleak --- Python/ceval.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/ceval.c b/Python/ceval.c index b0102b7cd4a7c9..dfdad3501296ad 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4904,6 +4904,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_XDECREF(kwargs); Py_DECREF(callargs); + Py_DECREF(func); _Py_LeaveRecursiveCallTstate(tstate); assert(TOP() == NULL); SET_TOP(res); From 92fead82dbf0680eeacaa970aed957a892e97d31 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:52:38 -0700 Subject: [PATCH 04/17] re-order STACK_SHRINK to fix error case --- Python/ceval.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index dfdad3501296ad..9e4b3746e33cbb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4838,7 +4838,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); - STACK_SHRINK(3); + STACK_SHRINK(3); /* Get rid of callargs, func and NULL */ STAT_INC(CALL_FUNCTION_EX, hit); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; @@ -4886,7 +4886,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DEOPT_IF(kwargs != NULL && ((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); STAT_INC(CALL_FUNCTION_EX, hit); - STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ // This is slower but CPython promises to check all non-vectorcall // function calls. @@ -4894,6 +4893,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; } + STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ /* res = func(self, args, nargs, kwnames) */ PyCFunction cfunc = PyCFunction_GET_FUNCTION(func); PyObject *res = ((PyCFunctionWithKeywords)(void(*)(void))cfunc)( From 84d0f8b737cca775a08bfcff2455ab8debae59fa Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:04:47 -0700 Subject: [PATCH 05/17] Inline instead of specializing --- Include/internal/pycore_opcode.h | 67 +++--- Include/opcode.h | 86 ++++---- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 11 - ...2-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 3 +- Python/ceval.c | 205 ++++-------------- Python/opcode_targets.h | 58 ++--- Python/specialize.c | 74 ------- 8 files changed, 152 insertions(+), 355 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 88feb7d693914b..15925511cc1f41 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -50,7 +50,6 @@ const uint8_t _PyOpcode_Caches[256] = { [COMPARE_OP] = 2, [LOAD_GLOBAL] = 5, [BINARY_OP] = 1, - [CALL_FUNCTION_EX] = 4, [CALL] = 4, }; @@ -89,10 +88,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_ADAPTIVE] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_PY_KWARGS] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_PY_NO_KWARGS] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, @@ -302,36 +297,36 @@ static const char *const _PyOpcode_OpName[263] = { [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [CALL_FUNCTION_EX_ADAPTIVE] = "CALL_FUNCTION_EX_ADAPTIVE", - [CALL_FUNCTION_EX_PY_KWARGS] = "CALL_FUNCTION_EX_PY_KWARGS", - [CALL_FUNCTION_EX_PY_NO_KWARGS] = "CALL_FUNCTION_EX_PY_NO_KWARGS", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", [COMPARE_OP_ADAPTIVE] = "COMPARE_OP_ADAPTIVE", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", + [STORE_SUBSCR] = "STORE_SUBSCR", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [PRINT_EXPR] = "PRINT_EXPR", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", [FOR_ITER_LIST] = "FOR_ITER_LIST", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", + [GET_ITER] = "GET_ITER", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [PRINT_EXPR] = "PRINT_EXPR", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [RETURN_GENERATOR] = "RETURN_GENERATOR", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -358,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -366,7 +361,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -386,9 +381,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -398,30 +393,26 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LIST_EXTEND] = "LIST_EXTEND", - [SET_UPDATE] = "SET_UPDATE", - [DICT_MERGE] = "DICT_MERGE", - [DICT_UPDATE] = "DICT_UPDATE", [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [RESUME_QUICK] = "RESUME_QUICK", + [LIST_EXTEND] = "LIST_EXTEND", + [SET_UPDATE] = "SET_UPDATE", + [DICT_MERGE] = "DICT_MERGE", + [DICT_UPDATE] = "DICT_UPDATE", [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [CALL] = "CALL", + [KW_NAMES] = "KW_NAMES", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", @@ -430,6 +421,10 @@ static const char *const _PyOpcode_OpName[263] = { [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [181] = "<181>", + [182] = "<182>", + [183] = "<183>", + [184] = "<184>", [185] = "<185>", [186] = "<186>", [187] = "<187>", @@ -512,6 +507,10 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 181: \ + case 182: \ + case 183: \ + case 184: \ case 185: \ case 186: \ case 187: \ diff --git a/Include/opcode.h b/Include/opcode.h index 50958474c5acef..42825df6217b46 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -158,51 +158,47 @@ extern "C" { #define CALL_NO_KW_STR_1 47 #define CALL_NO_KW_TUPLE_1 48 #define CALL_NO_KW_TYPE_1 56 -#define CALL_FUNCTION_EX_ADAPTIVE 57 -#define CALL_FUNCTION_EX_PY_KWARGS 58 -#define CALL_FUNCTION_EX_PY_NO_KWARGS 59 -#define CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS 62 -#define COMPARE_OP_ADAPTIVE 63 -#define COMPARE_OP_FLOAT_JUMP 64 -#define COMPARE_OP_INT_JUMP 65 -#define COMPARE_OP_STR_JUMP 66 -#define EXTENDED_ARG_QUICK 67 -#define FOR_ITER_ADAPTIVE 72 -#define FOR_ITER_LIST 73 -#define FOR_ITER_RANGE 76 -#define JUMP_BACKWARD_QUICK 77 -#define LOAD_ATTR_ADAPTIVE 78 -#define LOAD_ATTR_CLASS 79 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 80 -#define LOAD_ATTR_INSTANCE_VALUE 81 -#define LOAD_ATTR_MODULE 86 -#define LOAD_ATTR_PROPERTY 113 -#define LOAD_ATTR_SLOT 121 -#define LOAD_ATTR_WITH_HINT 141 -#define LOAD_ATTR_METHOD_LAZY_DICT 143 -#define LOAD_ATTR_METHOD_NO_DICT 153 -#define LOAD_ATTR_METHOD_WITH_DICT 154 -#define LOAD_ATTR_METHOD_WITH_VALUES 158 -#define LOAD_CONST__LOAD_FAST 159 -#define LOAD_FAST__LOAD_CONST 160 -#define LOAD_FAST__LOAD_FAST 161 -#define LOAD_GLOBAL_ADAPTIVE 166 -#define LOAD_GLOBAL_BUILTIN 167 -#define LOAD_GLOBAL_MODULE 168 -#define RESUME_QUICK 169 -#define STORE_ATTR_ADAPTIVE 170 -#define STORE_ATTR_INSTANCE_VALUE 173 -#define STORE_ATTR_SLOT 174 -#define STORE_ATTR_WITH_HINT 175 -#define STORE_FAST__LOAD_FAST 176 -#define STORE_FAST__STORE_FAST 177 -#define STORE_SUBSCR_ADAPTIVE 178 -#define STORE_SUBSCR_DICT 179 -#define STORE_SUBSCR_LIST_INT 180 -#define UNPACK_SEQUENCE_ADAPTIVE 181 -#define UNPACK_SEQUENCE_LIST 182 -#define UNPACK_SEQUENCE_TUPLE 183 -#define UNPACK_SEQUENCE_TWO_TUPLE 184 +#define COMPARE_OP_ADAPTIVE 57 +#define COMPARE_OP_FLOAT_JUMP 58 +#define COMPARE_OP_INT_JUMP 59 +#define COMPARE_OP_STR_JUMP 62 +#define EXTENDED_ARG_QUICK 63 +#define FOR_ITER_ADAPTIVE 64 +#define FOR_ITER_LIST 65 +#define FOR_ITER_RANGE 66 +#define JUMP_BACKWARD_QUICK 67 +#define LOAD_ATTR_ADAPTIVE 72 +#define LOAD_ATTR_CLASS 73 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 +#define LOAD_ATTR_INSTANCE_VALUE 77 +#define LOAD_ATTR_MODULE 78 +#define LOAD_ATTR_PROPERTY 79 +#define LOAD_ATTR_SLOT 80 +#define LOAD_ATTR_WITH_HINT 81 +#define LOAD_ATTR_METHOD_LAZY_DICT 86 +#define LOAD_ATTR_METHOD_NO_DICT 113 +#define LOAD_ATTR_METHOD_WITH_DICT 121 +#define LOAD_ATTR_METHOD_WITH_VALUES 141 +#define LOAD_CONST__LOAD_FAST 143 +#define LOAD_FAST__LOAD_CONST 153 +#define LOAD_FAST__LOAD_FAST 154 +#define LOAD_GLOBAL_ADAPTIVE 158 +#define LOAD_GLOBAL_BUILTIN 159 +#define LOAD_GLOBAL_MODULE 160 +#define RESUME_QUICK 161 +#define STORE_ATTR_ADAPTIVE 166 +#define STORE_ATTR_INSTANCE_VALUE 167 +#define STORE_ATTR_SLOT 168 +#define STORE_ATTR_WITH_HINT 169 +#define STORE_FAST__LOAD_FAST 170 +#define STORE_FAST__STORE_FAST 173 +#define STORE_SUBSCR_ADAPTIVE 174 +#define STORE_SUBSCR_DICT 175 +#define STORE_SUBSCR_LIST_INT 176 +#define UNPACK_SEQUENCE_ADAPTIVE 177 +#define UNPACK_SEQUENCE_LIST 178 +#define UNPACK_SEQUENCE_TUPLE 179 +#define UNPACK_SEQUENCE_TWO_TUPLE 180 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4db6f3faf7857d..734fe2dce9fb51 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -413,7 +413,6 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3507 (Set lineno of module's RESUME to 0) # Python 3.12a1 3508 (Add CLEANUP_THROW) # Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a1 3510 (Specialization/Cache for CALL_FUNCTION_EX) # Python 3.13 will start with 3550 @@ -427,7 +426,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3510).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3509).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index bd48c594851084..690923061418bd 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -311,12 +311,6 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TUPLE_1", "CALL_NO_KW_TYPE_1", ], - "CALL_FUNCTION_EX": [ - "CALL_FUNCTION_EX_ADAPTIVE", - "CALL_FUNCTION_EX_PY_KWARGS", - "CALL_FUNCTION_EX_PY_NO_KWARGS", - "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", - ], "COMPARE_OP": [ "COMPARE_OP_ADAPTIVE", "COMPARE_OP_FLOAT_JUMP", @@ -440,11 +434,6 @@ def pseudo_op(name, op, real_ops): "func_version": 2, "min_args": 1, }, - "CALL_FUNCTION_EX": { - "counter": 1, - "func_version": 2, - "min_args": 1, - }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst index a6dc08f6092ba3..c0764d245132e7 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -1 +1,2 @@ -Specialize complex calls. +Complex function calls are now significantly faster and consume no C stack +space. \ No newline at end of file diff --git a/Python/ceval.c b/Python/ceval.c index 9e4b3746e33cbb..36cb0c993180de 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4741,177 +4741,64 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - - STACK_SHRINK(1); - assert(TOP() == NULL); - SET_TOP(result); - if (result == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_FUNCTION_EX_ADAPTIVE) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - PyObject *func, *callargs, *kwargs = NULL; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - if (oparg & 0x01) { - kwargs = TOP(); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - callargs = SECOND(); - func = THIRD(); + if (Py_IS_TYPE(func, &PyFunction_Type) && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + goto error; } - else { - callargs = TOP(); - func = SECOND(); + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } } - - int err = _Py_Specialize_CallEx(func, next_instr, callargs, kwargs); - if (err < 0) { + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + STACK_SHRINK(2); /* get rid of func and NULL */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + if (new_frame == NULL) { goto error; } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(CALL_FUNCTION_EX, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - JUMP_TO_INSTRUCTION(CALL_FUNCTION_EX); - } - } - - TARGET(CALL_FUNCTION_EX_PY_KWARGS) { - assert(oparg & 0x01); - PyObject *func, *callargs, *kwargs = NULL; - kwargs = TOP(); - callargs = SECOND(); - func = THIRD(); - DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - DEOPT_IF(kwargs == NULL || PyDict_GET_SIZE(kwargs) <= 0, CALL_FUNCTION_EX); - DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); - DEOPT_IF(((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); - STAT_INC(CALL_FUNCTION_EX, hit); - STACK_SHRINK(4); /* Get rid of kwargs, callargs, func and NULL */ - - PyObject *kwnames; - PyObject *const *newargs = _PyStack_UnpackDict(tstate, - _PyTuple_ITEMS(callargs), nargs, - kwargs, &kwnames); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; - CALL_STAT_INC(frames_pushed); - - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); - Py_DECREF(kwargs); - Py_DECREF(callargs); - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - if (new_frame == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; - } - - TARGET(CALL_FUNCTION_EX_PY_NO_KWARGS) { - assert(!(oparg & 0x01)); - PyObject *func, *callargs; - callargs = TOP(); - func = SECOND(); - DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); - STACK_SHRINK(3); /* Get rid of callargs, func and NULL */ - STAT_INC(CALL_FUNCTION_EX, hit); - - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; - CALL_STAT_INC(frames_pushed); - - for (Py_ssize_t i = 0; i < nargs; ++i) { - /* We need to incref all our args since the new frame steals the references. */ - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - &PyTuple_GET_ITEM(callargs, 0), nargs, NULL - ); - Py_DECREF(callargs); - if (new_frame == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; - } - - TARGET(CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS) { - PyObject *func, *callargs, *kwargs = NULL; - int has_kwargs = oparg & 0x01; - if (has_kwargs) { - kwargs = TOP(); - callargs = SECOND(); - func = THIRD(); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; } else { - callargs = TOP(); - func = SECOND(); + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); } - DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - DEOPT_IF(!PyCFunction_CheckExact(func), CALL_FUNCTION_EX); - DEOPT_IF(PyCFunction_GET_FLAGS(func) != (METH_VARARGS | METH_KEYWORDS), CALL); - DEOPT_IF(kwargs != NULL && - ((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); - STAT_INC(CALL_FUNCTION_EX, hit); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - - STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ - /* res = func(self, args, nargs, kwnames) */ - PyCFunction cfunc = PyCFunction_GET_FUNCTION(func); - PyObject *res = ((PyCFunctionWithKeywords)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(func), - callargs, - (kwargs == NULL || PyDict_GET_SIZE(kwargs) == 0) ? NULL : kwargs - ); - Py_XDECREF(kwargs); - Py_DECREF(callargs); - Py_DECREF(func); - _Py_LeaveRecursiveCallTstate(tstate); + STACK_SHRINK(1); assert(TOP() == NULL); - SET_TOP(res); - if (res == NULL) { + SET_TOP(result); + if (result == NULL) { goto error; } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 32cbf25a9929bd..c1ff367d4fd38d 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -56,36 +56,36 @@ static void *opcode_targets[256] = { &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_CALL_FUNCTION_EX_ADAPTIVE, - &&TARGET_CALL_FUNCTION_EX_PY_KWARGS, - &&TARGET_CALL_FUNCTION_EX_PY_NO_KWARGS, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, - &&TARGET_CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_INT_JUMP, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_EXTENDED_ARG_QUICK, - &&TARGET_GET_ITER, - &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_PRINT_EXPR, - &&TARGET_LOAD_BUILD_CLASS, &&TARGET_FOR_ITER_ADAPTIVE, &&TARGET_FOR_ITER_LIST, - &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_RETURN_GENERATOR, &&TARGET_FOR_ITER_RANGE, &&TARGET_JUMP_BACKWARD_QUICK, + &&TARGET_GET_ITER, + &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_PRINT_EXPR, + &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_RETURN_GENERATOR, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,30 +152,26 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_LIST_EXTEND, - &&TARGET_SET_UPDATE, - &&TARGET_DICT_MERGE, - &&TARGET_DICT_UPDATE, &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_RESUME_QUICK, + &&TARGET_LIST_EXTEND, + &&TARGET_SET_UPDATE, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_CALL, - &&TARGET_KW_NAMES, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_CALL, + &&TARGET_KW_NAMES, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, @@ -254,5 +250,9 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index d012ba4b119df1..b7c321e4878b98 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -24,7 +24,6 @@ uint8_t _PyOpcode_Adaptive[256] = { [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, [CALL] = CALL_ADAPTIVE, - [CALL_FUNCTION_EX] = CALL_FUNCTION_EX_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, [BINARY_OP] = BINARY_OP_ADAPTIVE, [COMPARE_OP] = COMPARE_OP_ADAPTIVE, @@ -1801,79 +1800,6 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return 0; } - -static int -specialize_py_callex(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs, int oparg) -{ - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0) { - assert(oparg & 0x01); - if (((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - /* kwargs keys should be unicode */ - SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); - return 1; - } - _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_KWARGS); - return 0; - } - else { - if (oparg & 0x01) { - /* Bad TOS dict or something */ - SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); - return 1; - } - _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_NO_KWARGS); - return 0; - } -} - - -int -_Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs) -{ - int oparg = _Py_OPARG(*instr); - assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); - int fail; - if (!PyTuple_CheckExact(args)) { - fail = 1; - goto fail_please; - } - if (Py_IS_TYPE(callable, &PyFunction_Type)) { - fail = specialize_py_callex(callable, instr, args, kwargs, oparg); - } - else if (PyCFunction_CheckExact(callable)) { - PyCFunctionObject *func = (PyCFunctionObject *)callable; - switch (func->m_ml->ml_flags & - (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS | METH_KEYWORDS: - _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS); - break; - default: - fail = 1; - } - } - else { - SPECIALIZATION_FAIL(CALL_FUNCTION_EX, call_fail_kind(callable)); - fail = 1; - } - -fail_please: - if (fail) { - STAT_INC(CALL_FUNCTION_EX, failure); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(CALL_FUNCTION_EX, success); - assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); - } - return 0; -} - - #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) From 1955f19e0156cf032fe0d8e3c6273998d5c82eb9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:05:27 -0700 Subject: [PATCH 06/17] reduce diff --- Lib/importlib/_bootstrap_external.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 734fe2dce9fb51..b3c31b9659d849 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -414,7 +414,6 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3508 (Add CLEANUP_THROW) # Python 3.12a1 3509 (Conditional jumps only jump forward) - # Python 3.13 will start with 3550 # MAGIC must change whenever the bytecode emitted by the compiler may no From eb25c1bceedc495cb9dac882d55cd3ec047f7248 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:32:54 -0700 Subject: [PATCH 07/17] temper expectations --- .../2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst index c0764d245132e7..efa56f87c8be8c 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -1,2 +1,2 @@ -Complex function calls are now significantly faster and consume no C stack -space. \ No newline at end of file +Complex function calls are now faster and consume no C stack +space. From 3f409da31683fa7b7d2fe68731a742bd26db3a26 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 16:20:54 -0700 Subject: [PATCH 08/17] Add newline --- .../2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst index efa56f87c8be8c..f9e71bc1344bb3 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -1,2 +1,3 @@ Complex function calls are now faster and consume no C stack space. + From 09b25281645858127a073f931d9eada81eaedce7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Oct 2022 11:27:26 +0800 Subject: [PATCH 09/17] remove unused variable --- Python/ceval.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 36cb0c993180de..08f23e14682b67 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4748,7 +4748,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; From f0d77dc3080c9c9914e4fd1e43b8c0ad1eaa3c8a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 24 Nov 2022 23:52:59 +0800 Subject: [PATCH 10/17] remove unused function prototype --- Include/internal/pycore_code.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 29caf4d524c145..bf5945435c1774 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -244,7 +244,6 @@ extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_C extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames); -extern int _Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, From b933f6e0ea4d5eef05bf7b90c8d8f8f8af061c09 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 24 Nov 2022 23:59:02 +0800 Subject: [PATCH 11/17] Use new case regenerator --- Python/bytecodes.c | 53 +++++++++++++++++++++++++++++++++++--- Python/generated_cases.c.h | 53 +++++++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1f910da8ed54a..d39c529ad37020 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3475,10 +3475,55 @@ dummy_func( } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + if (Py_IS_TYPE(func, &PyFunction_Type) && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + goto error; + } + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + STACK_SHRINK(2); /* get rid of func and NULL */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + else { + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + } STACK_SHRINK(1); assert(TOP() == NULL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ae8fdd5e99c3dc..9139a716ad8f3f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3499,10 +3499,55 @@ } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + if (Py_IS_TYPE(func, &PyFunction_Type) && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + goto error; + } + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + STACK_SHRINK(2); /* get rid of func and NULL */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + else { + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + } STACK_SHRINK(1); assert(TOP() == NULL); From 06ddc00a8f5063fde60ba0c548c4a3a69a162ec6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 1 Dec 2022 01:47:32 +0800 Subject: [PATCH 12/17] Apply Mark's and Brandt's suggestion --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Python/bytecodes.c | 34 ++------------ Python/ceval.c | 45 +++++++++++++++++++ Python/generated_cases.c.h | 34 ++------------ 7 files changed, 58 insertions(+), 60 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 494bcf293cdb7b..0919fdd668b858 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1218,6 +1218,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zipimporter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[2]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b0cb8365933e77..8abb561c053505 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -704,6 +704,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) + STRUCT_FOR_ID(zipimporter) } identifiers; struct { PyASCIIObject _ascii; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4b128da54555b7..5c28d04b5f5768 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1210,6 +1210,7 @@ extern "C" { INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ + INIT_ID(zipimporter), \ } #define _Py_str_ascii_INIT { \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7ef1f7e94ddead..68c757a180c73c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1314,6 +1314,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(zdict); PyUnicode_InternInPlace(&string); + string = &_Py_ID(zipimporter); + PyUnicode_InternInPlace(&string); } /* End auto-generated code */ #ifdef __cplusplus diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d39c529ad37020..7866f24eab20cf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3483,40 +3483,14 @@ dummy_func( int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - PyObject *const *newargs = has_dict - ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), - nargs, kwargs, &kwnames) - : &PyTuple_GET_ITEM(callargs, 0); - if (newargs == NULL) { - goto error; - } - if (!has_dict) { - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); STACK_SHRINK(2); /* get rid of func and NULL */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - } if (new_frame == NULL) { goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } else { result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); diff --git a/Python/ceval.c b/Python/ceval.c index 80bfa21ad0b6f0..4e13b2f45e1714 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -217,6 +217,9 @@ static _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames); +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -2005,6 +2008,48 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return NULL; } +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) +{ + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + Py_DECREF(func); + goto error; + } + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; +} + static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9139a716ad8f3f..8ac5e0ffa0ee73 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3507,40 +3507,14 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - PyObject *const *newargs = has_dict - ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), - nargs, kwargs, &kwnames) - : &PyTuple_GET_ITEM(callargs, 0); - if (newargs == NULL) { - goto error; - } - if (!has_dict) { - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); STACK_SHRINK(2); /* get rid of func and NULL */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - } if (new_frame == NULL) { goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } else { result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); From bfab4dcffa6e8cf3012fa07e70a8db1b2a9e983a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 1 Dec 2022 23:09:58 +0800 Subject: [PATCH 13/17] Run --regen --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 2 -- 4 files changed, 5 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0919fdd668b858..494bcf293cdb7b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1218,7 +1218,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zipimporter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[2]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8abb561c053505..b0cb8365933e77 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -704,7 +704,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) - STRUCT_FOR_ID(zipimporter) } identifiers; struct { PyASCIIObject _ascii; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5c28d04b5f5768..4b128da54555b7 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1210,7 +1210,6 @@ extern "C" { INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ - INIT_ID(zipimporter), \ } #define _Py_str_ascii_INIT { \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 68c757a180c73c..7ef1f7e94ddead 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1314,8 +1314,6 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(zdict); PyUnicode_InternInPlace(&string); - string = &_Py_ID(zipimporter); - PyUnicode_InternInPlace(&string); } /* End auto-generated code */ #ifdef __cplusplus From dfd6b01e99fbb4830d015a8e3e018e5c920ba486 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 27 Apr 2023 17:34:38 +0800 Subject: [PATCH 14/17] Fix merge problems Co-Authored-By: Brandt Bucher --- Python/bytecodes.c | 4 +++- Python/ceval.c | 1 + Python/generated_cases.c.h | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cfce9da3d77376..968e912545f1e6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3114,10 +3114,12 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - STACK_SHRINK(2); /* get rid of func and NULL */ + // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; } + frame->return_offset = 0; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); diff --git a/Python/ceval.c b/Python/ceval.c index 5597ff4a67219b..958689debc87f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1513,6 +1513,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; + PyObject *const *newargs; if (has_dict) { newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); if (newargs == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 026b27c1f4106a..d19de3aed2e538 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3860,10 +3860,11 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - STACK_SHRINK(2); /* get rid of func and NULL */ + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; } + frame->return_offset = 0; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); From 1afff6dc586b596db6a9884a37f75c4da937715e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 28 Apr 2023 21:15:15 +0800 Subject: [PATCH 15/17] Update generated_cases.c.h --- Python/generated_cases.c.h | 491 +++++++++++++++++++++++++++++++++++++ 1 file changed, 491 insertions(+) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d19de3aed2e538..153a6325534ac2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,7 @@ } TARGET(RESUME) { + #line 137 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ @@ -19,10 +20,12 @@ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } + #line 24 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_RESUME) { + #line 151 "Python/bytecodes.c" /* Possible performance enhancement: * We need to check the eval breaker anyway, can we * combine the instrument verison check and the eval breaker test? @@ -48,15 +51,18 @@ goto handle_eval_breaker; } } + #line 55 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; + #line 179 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); + #line 66 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -64,9 +70,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; + #line 186 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); + #line 78 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -74,9 +82,11 @@ TARGET(LOAD_FAST) { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 90 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -85,8 +95,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; + #line 198 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 102 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -94,7 +106,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 112 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -104,17 +118,21 @@ PyObject *_tmp_2; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 126 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 136 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -128,16 +146,20 @@ PyObject *_tmp_2; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 154 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 198 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 163 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -150,14 +172,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 178 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 187 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -169,12 +195,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 201 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 208 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -185,16 +215,20 @@ PyObject *_tmp_2; { PyObject *value; + #line 198 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 222 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 232 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -205,6 +239,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; + #line 213 "Python/bytecodes.c" + #line 244 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -212,7 +248,9 @@ TARGET(PUSH_NULL) { PyObject *res; + #line 217 "Python/bytecodes.c" res = NULL; + #line 254 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -223,10 +261,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; + #line 213 "Python/bytecodes.c" + #line 266 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; + #line 213 "Python/bytecodes.c" + #line 272 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -236,6 +278,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 223 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -245,6 +288,7 @@ } PyErr_SetRaisedException(NULL); } + #line 292 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -254,7 +298,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 236 "Python/bytecodes.c" Py_DECREF(receiver); + #line 304 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -263,6 +309,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 240 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -271,6 +318,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); + #line 322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -279,9 +327,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 251 "Python/bytecodes.c" res = PyNumber_Negative(value); + #line 333 "Python/generated_cases.c.h" Py_DECREF(value); + #line 253 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 337 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -289,8 +341,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 257 "Python/bytecodes.c" int err = PyObject_IsTrue(value); + #line 347 "Python/generated_cases.c.h" Py_DECREF(value); + #line 259 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -299,6 +354,7 @@ res = Py_False; } Py_INCREF(res); + #line 358 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -306,9 +362,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 270 "Python/bytecodes.c" res = PyNumber_Invert(value); + #line 368 "Python/generated_cases.c.h" Py_DECREF(value); + #line 272 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 372 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -317,6 +377,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; + #line 289 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -324,6 +385,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; + #line 389 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -334,12 +396,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; + #line 299 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); + #line 407 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -350,6 +414,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; + #line 308 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -357,6 +422,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; + #line 426 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -367,11 +433,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; + #line 318 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); + #line 443 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -382,6 +450,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 326 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -389,6 +458,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; + #line 462 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -398,6 +468,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 342 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -424,6 +495,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + #line 499 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -432,12 +504,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; + #line 371 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); + #line 515 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -448,6 +522,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; + #line 380 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -455,6 +530,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; + #line 534 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -467,6 +543,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; + #line 398 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -478,9 +555,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); + #line 559 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); + #line 410 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 564 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -492,6 +572,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; + #line 414 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -504,6 +585,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; + #line 589 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -514,6 +596,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; + #line 429 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -526,6 +609,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; + #line 613 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -534,6 +618,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; + #line 444 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -547,6 +632,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); + #line 636 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -557,6 +643,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; + #line 460 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -570,6 +657,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); + #line 661 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -580,6 +668,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; + #line 476 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -587,11 +676,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } + #line 680 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); + #line 484 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub + #line 687 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -603,6 +695,7 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; + #line 491 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -624,12 +717,15 @@ JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 721 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; + #line 515 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + #line 729 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -638,9 +734,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; + #line 520 "Python/bytecodes.c" int err = PySet_Add(set, v); + #line 740 "Python/generated_cases.c.h" Py_DECREF(v); + #line 522 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 744 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -653,6 +753,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); + #line 533 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -667,10 +768,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); + #line 772 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); + #line 548 "Python/bytecodes.c" if (err) goto pop_3_error; + #line 778 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -680,6 +784,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; + #line 552 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -696,6 +801,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); + #line 805 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -705,11 +811,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; + #line 571 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; + #line 821 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -718,11 +826,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; + #line 579 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); + #line 833 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); + #line 582 "Python/bytecodes.c" if (err) goto pop_2_error; + #line 838 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -730,10 +842,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 586 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); + #line 849 "Python/generated_cases.c.h" Py_DECREF(value); + #line 589 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 853 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -742,11 +858,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; + #line 593 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + #line 865 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); + #line 596 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 870 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -754,6 +874,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); + #line 600 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -771,10 +892,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } + #line 896 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; + #line 620 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -785,10 +908,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; + #line 912 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 633 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -801,10 +926,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 930 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 648 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -821,9 +948,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 952 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { + #line 667 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -837,9 +966,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 970 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { + #line 683 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -857,11 +988,13 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 992 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; + #line 703 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -874,12 +1007,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + #line 1011 "Python/generated_cases.c.h" Py_DECREF(obj); + #line 716 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); + #line 1018 "Python/generated_cases.c.h" Py_DECREF(obj); + #line 721 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -892,6 +1029,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } + #line 1033 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -899,6 +1037,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; + #line 736 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -942,6 +1081,7 @@ } } + #line 1085 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -952,13 +1092,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 783 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } + #line 1103 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 790 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -976,6 +1119,7 @@ if (iter == NULL) goto pop_1_error; + #line 1123 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -986,6 +1130,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; + #line 816 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1031,6 +1176,7 @@ } } Py_DECREF(v); + #line 1180 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1039,6 +1185,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 864 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1053,10 +1200,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND); DISPATCH_INLINED(gen_frame); + #line 1204 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 881 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1073,10 +1222,12 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 1226 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 900 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1092,12 +1243,15 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 1247 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; + #line 918 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); + #line 1255 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1105,6 +1259,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); + #line 923 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1122,21 +1277,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; + #line 1281 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; + #line 943 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + #line 1290 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); + #line 946 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } + #line 1300 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1147,19 +1307,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; + #line 955 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + #line 1316 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); + #line 960 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } + #line 1327 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1168,7 +1332,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; + #line 969 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); + #line 1338 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1176,6 +1342,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; + #line 973 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1197,6 +1364,7 @@ if (true) goto error; } } + #line 1368 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1204,26 +1372,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; + #line 997 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); + #line 1383 "Python/generated_cases.c.h" Py_DECREF(v); + #line 1004 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + #line 1392 "Python/generated_cases.c.h" Py_DECREF(v); + #line 1011 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1396 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { + #line 1015 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1240,6 +1415,7 @@ name); goto error; } + #line 1419 "Python/generated_cases.c.h" DISPATCH(); } @@ -1247,6 +1423,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; + #line 1041 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1259,8 +1436,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); + #line 1440 "Python/generated_cases.c.h" Py_DECREF(seq); + #line 1054 "Python/bytecodes.c" if (res == 0) goto pop_1_error; + #line 1444 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1270,12 +1450,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 1058 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + #line 1461 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1286,6 +1468,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 1068 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1293,6 +1476,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } + #line 1480 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1303,6 +1487,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 1079 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1310,6 +1495,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } + #line 1499 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1319,11 +1505,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; + #line 1090 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + #line 1513 "Python/generated_cases.c.h" Py_DECREF(seq); + #line 1094 "Python/bytecodes.c" if (res == 0) goto pop_1_error; + #line 1517 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1334,6 +1524,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); + #line 1105 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1349,9 +1540,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); + #line 1544 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); + #line 1121 "Python/bytecodes.c" if (err) goto pop_2_error; + #line 1549 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1359,25 +1553,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; + #line 1125 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + #line 1560 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1128 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1564 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; + #line 1132 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); + #line 1574 "Python/generated_cases.c.h" Py_DECREF(v); + #line 1135 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1578 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { + #line 1139 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1389,11 +1592,13 @@ } goto error; } + #line 1596 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; + #line 1153 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1452,6 +1657,7 @@ } } } + #line 1661 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1462,6 +1668,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; + #line 1220 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1513,6 +1720,7 @@ } } null = NULL; + #line 1724 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1526,6 +1734,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); + #line 1274 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1536,6 +1745,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; + #line 1749 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1550,6 +1760,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); + #line 1287 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1564,6 +1775,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; + #line 1779 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1573,13 +1785,16 @@ } TARGET(DELETE_FAST) { + #line 1304 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); + #line 1793 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { + #line 1310 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1588,10 +1803,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); + #line 1807 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { + #line 1321 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1602,11 +1819,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); + #line 1823 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; + #line 1334 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1638,6 +1857,7 @@ } Py_INCREF(value); } + #line 1861 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1645,6 +1865,7 @@ TARGET(LOAD_DEREF) { PyObject *value; + #line 1368 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1652,6 +1873,7 @@ if (true) goto error; } Py_INCREF(value); + #line 1877 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1659,15 +1881,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; + #line 1378 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); + #line 1890 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { + #line 1385 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1678,17 +1903,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } + #line 1907 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; + #line 1398 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + #line 1916 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } + #line 1400 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1922 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1698,8 +1928,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; + #line 1404 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1935 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1709,8 +1941,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; + #line 1409 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1948 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1720,6 +1954,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; + #line 1414 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1730,10 +1965,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } + #line 1969 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 1425 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); + #line 1975 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1742,9 +1980,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; + #line 1432 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); + #line 1986 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 1434 "Python/bytecodes.c" if (err < 0) goto pop_1_error; + #line 1990 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1752,6 +1994,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; + #line 1438 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -1766,6 +2009,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } + #line 2013 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -1775,6 +2019,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; + #line 1455 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1782,10 +2027,13 @@ if (map == NULL) goto error; + #line 2031 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } + #line 1463 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } + #line 2037 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -1793,6 +2041,7 @@ } TARGET(SETUP_ANNOTATIONS) { + #line 1467 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -1832,6 +2081,7 @@ Py_DECREF(ann_dict); } } + #line 2085 "Python/generated_cases.c.h" DISPATCH(); } @@ -1839,6 +2089,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; + #line 1509 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1848,11 +2099,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + #line 2103 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); + #line 1519 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } + #line 2110 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -1860,6 +2114,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; + #line 1523 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1867,9 +2122,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } + #line 2126 "Python/generated_cases.c.h" Py_DECREF(update); + #line 1531 "Python/bytecodes.c" if (true) goto pop_1_error; } + #line 2131 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -1877,13 +2135,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; + #line 1537 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); + #line 1542 "Python/bytecodes.c" if (true) goto pop_1_error; } + #line 2149 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -1893,11 +2155,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; + #line 1549 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + #line 2165 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -1911,6 +2175,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; + #line 1563 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -1928,13 +2193,16 @@ // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + #line 2197 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); + #line 1581 "Python/bytecodes.c" if (super == NULL) goto pop_3_error; res = PyObject_GetAttr(super, name); Py_DECREF(super); if (res == NULL) goto pop_3_error; + #line 2206 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1952,6 +2220,7 @@ uint32_t class_version = read_u32(&next_instr[1].cache); uint32_t self_type_version = read_u32(&next_instr[3].cache); PyObject *method = read_obj(&next_instr[5].cache); + #line 1588 "Python/bytecodes.c" DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); DEOPT_IF(((PyTypeObject *)class)->tp_version_tag != class_version, LOAD_SUPER_ATTR); @@ -1962,6 +2231,7 @@ Py_INCREF(res2); Py_DECREF(global_super); Py_DECREF(class); + #line 2235 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -1975,6 +2245,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; + #line 1615 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2008,7 +2279,9 @@ NULL | meth | arg1 | ... | argN */ + #line 2283 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1649 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2017,9 +2290,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); + #line 2294 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1658 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } + #line 2299 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2033,6 +2309,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1663 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2045,6 +2322,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2326 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2059,6 +2337,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1679 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2071,6 +2350,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2354 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2085,6 +2365,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1695 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2111,6 +2392,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2396 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2125,6 +2407,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1725 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2134,6 +2417,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2421 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2148,6 +2432,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 1738 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2159,6 +2444,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); + #line 2448 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2172,6 +2458,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); + #line 1753 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2195,6 +2482,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 2486 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2202,6 +2490,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); + #line 1779 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2227,6 +2516,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 2520 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2234,6 +2524,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1807 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2251,6 +2542,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); + #line 2546 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2261,6 +2553,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); + #line 1827 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2299,6 +2592,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); + #line 2596 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2309,6 +2603,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1868 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2318,6 +2613,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); + #line 2617 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2329,6 +2625,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1887 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2341,9 +2638,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); + #line 2642 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1900 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 2647 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2354,6 +2654,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1904 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2365,6 +2666,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); + #line 2670 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2375,6 +2677,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1919 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2390,6 +2693,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); + #line 2697 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2400,6 +2704,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1938 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2412,6 +2717,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); + #line 2721 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2422,10 +2728,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1953 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; + #line 2734 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1955 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); + #line 2739 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2435,11 +2745,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1959 "Python/bytecodes.c" int res = PySequence_Contains(right, left); + #line 2751 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1961 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); + #line 2757 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2450,9 +2764,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; + #line 1966 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { + #line 2770 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); + #line 1968 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2460,8 +2777,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); + #line 2781 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); + #line 1976 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2470,6 +2789,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } + #line 2793 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2479,15 +2799,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1987 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { + #line 2806 "Python/generated_cases.c.h" Py_DECREF(right); + #line 1990 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); + #line 2813 "Python/generated_cases.c.h" Py_DECREF(right); + #line 1995 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); + #line 2817 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2496,11 +2822,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; + #line 1999 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); + #line 2829 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); + #line 2002 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 2834 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2509,23 +2839,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; + #line 2006 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; + #line 2847 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { + #line 2012 "Python/bytecodes.c" JUMPBY(oparg); + #line 2856 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); + #line 2016 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); + #line 2865 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2533,6 +2869,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; + #line 2022 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2542,7 +2879,9 @@ } else { int err = PyObject_IsTrue(cond); + #line 2883 "Python/generated_cases.c.h" Py_DECREF(cond); + #line 2032 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2550,12 +2889,14 @@ if (err < 0) goto pop_1_error; } } + #line 2893 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; + #line 2042 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2565,7 +2906,9 @@ } else { int err = PyObject_IsTrue(cond); + #line 2910 "Python/generated_cases.c.h" Py_DECREF(cond); + #line 2052 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2573,54 +2916,67 @@ if (err < 0) goto pop_1_error; } } + #line 2920 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; + #line 2062 "Python/bytecodes.c" if (!Py_IsNone(value)) { + #line 2929 "Python/generated_cases.c.h" Py_DECREF(value); + #line 2064 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } + #line 2937 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; + #line 2072 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { + #line 2950 "Python/generated_cases.c.h" Py_DECREF(value); + #line 2078 "Python/bytecodes.c" } + #line 2954 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #line 2082 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); + #line 2967 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; + #line 2091 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; + #line 2980 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2631,13 +2987,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; + #line 2099 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); + #line 2996 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); + #line 2104 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2645,6 +3004,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } + #line 3008 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2653,8 +3013,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; + #line 2114 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); + #line 3020 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2664,8 +3026,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; + #line 2120 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); + #line 3033 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2676,9 +3040,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; + #line 2126 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; + #line 3048 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2687,10 +3053,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 2132 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); + #line 3060 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 2135 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; + #line 3064 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2698,6 +3068,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 2139 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2720,8 +3091,11 @@ if (iter == NULL) { goto error; } + #line 3095 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 2162 "Python/bytecodes.c" } + #line 3099 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -2732,6 +3106,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2181 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2762,6 +3137,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator + #line 3141 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2769,6 +3145,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { + #line 2214 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -2794,12 +3171,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + #line 3175 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2242 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -2819,6 +3198,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator + #line 3202 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2828,6 +3208,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2264 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2847,6 +3228,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator + #line 3232 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2856,6 +3238,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2286 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2873,6 +3256,7 @@ if (next == NULL) { goto error; } + #line 3260 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2881,6 +3265,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; + #line 2306 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2895,12 +3280,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); + #line 3284 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; + #line 2323 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -2923,13 +3310,16 @@ Py_DECREF(enter); goto error; } + #line 3314 "Python/generated_cases.c.h" Py_DECREF(mgr); + #line 2346 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } + #line 3323 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -2941,6 +3331,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; + #line 2356 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -2966,13 +3357,16 @@ Py_DECREF(enter); goto error; } + #line 3361 "Python/generated_cases.c.h" Py_DECREF(mgr); + #line 2382 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } + #line 3370 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -2984,6 +3378,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; + #line 2391 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3004,6 +3399,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; + #line 3403 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3012,6 +3408,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; + #line 2414 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3021,6 +3418,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); + #line 3422 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3034,6 +3432,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2426 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3050,6 +3449,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); + #line 3453 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3063,6 +3463,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2445 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3072,6 +3473,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); + #line 3477 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3085,6 +3487,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2457 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3098,6 +3501,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); + #line 3505 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3106,13 +3510,16 @@ } TARGET(KW_NAMES) { + #line 2473 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); + #line 3518 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { + #line 2479 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3125,6 +3532,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); + #line 3536 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3134,6 +3542,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2524 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3215,6 +3624,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3628 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3226,6 +3636,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; + #line 2612 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3235,6 +3646,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + #line 3650 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3243,6 +3655,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); + #line 2624 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3268,6 +3681,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 3685 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3275,6 +3689,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); + #line 2652 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3310,6 +3725,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 3729 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3317,6 +3733,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2690 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3326,6 +3743,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable + #line 3747 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3338,6 +3756,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2702 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3348,6 +3767,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3771 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3361,6 +3781,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2716 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3371,6 +3792,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3796 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3384,6 +3806,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2730 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3405,6 +3828,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3832 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3418,6 +3842,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2755 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3445,6 +3870,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3874 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3458,6 +3884,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2786 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3489,6 +3916,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ + #line 3920 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3502,6 +3930,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2821 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3533,6 +3962,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3966 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3546,6 +3976,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2856 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3570,6 +4001,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4005 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3582,6 +4014,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2883 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3608,6 +4041,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4045 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3619,6 +4053,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; + #line 2913 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3636,12 +4071,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); + #line 4075 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2933 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3672,6 +4109,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4113 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3684,6 +4122,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2967 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3712,6 +4151,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4155 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3724,6 +4164,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2999 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -3752,6 +4193,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4197 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3764,6 +4206,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 3031 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3791,6 +4234,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4238 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3800,7 +4244,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #line 3062 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + #line 4250 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -3809,6 +4255,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; + #line 3066 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -3860,6 +4307,7 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); + // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; @@ -3869,11 +4317,14 @@ } result = PyObject_Call(func, callargs, kwargs); } + #line 4321 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); + #line 3128 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } + #line 4328 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -3888,6 +4339,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; + #line 3138 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -3916,12 +4368,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; + #line 4372 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { + #line 3169 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3942,6 +4396,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; + #line 4400 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -3949,11 +4404,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; + #line 3192 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); + #line 4410 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); + #line 3194 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } + #line 4416 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -3964,6 +4423,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; + #line 3198 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -3998,6 +4458,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } + #line 4462 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4006,8 +4467,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; + #line 3235 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); + #line 4474 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4019,6 +4482,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; + #line 3240 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4033,9 +4497,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); + #line 4501 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); + #line 3255 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 4506 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4045,13 +4512,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + #line 3260 "Python/bytecodes.c" assert(oparg >= 2); + #line 4518 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { + #line 3264 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4071,9 +4541,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); + #line 4545 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { + #line 3286 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4085,20 +4557,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); + #line 4561 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { + #line 3300 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + #line 4567 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { + #line 3304 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); + #line 4574 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + #line 3309 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4107,10 +4585,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4589 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + #line 3320 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4119,10 +4599,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4603 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + #line 3331 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4135,10 +4617,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4621 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + #line 3346 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4151,23 +4635,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4639 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { + #line 3361 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); + #line 4650 "Python/generated_cases.c.h" } TARGET(CACHE) { + #line 3369 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); + #line 4657 "Python/generated_cases.c.h" } TARGET(RESERVED) { + #line 3374 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); + #line 4664 "Python/generated_cases.c.h" } From f1a94659953a8674e1a7533dd02338ceb102985a Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 29 Apr 2023 12:21:11 +0800 Subject: [PATCH 16/17] Update Python/bytecodes.c Co-authored-by: Carl Meyer --- Python/bytecodes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 968e912545f1e6..e83894e8902872 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3114,7 +3114,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; From 94782c385e19183a8511334da97bb81ad5d9c0c1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 29 Apr 2023 15:53:10 +0800 Subject: [PATCH 17/17] Update generated_cases.c.h --- Python/generated_cases.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 153a6325534ac2..069a7ced0a4c25 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4307,7 +4307,7 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error;