From f4c2ce6b5416e5ce9150aeb5d1406dc5db149a21 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 17 Feb 2022 13:24:52 -0800 Subject: [PATCH 1/2] Specialize BINARY_OP_AND_INT --- Include/internal/pycore_long.h | 1 + Include/opcode.h | 123 +++++++++++++++++---------------- Lib/opcode.py | 1 + Objects/longobject.c | 6 ++ Python/ceval.c | 19 +++++ Python/opcode_targets.h | 38 +++++----- Python/specialize.c | 15 ++-- 7 files changed, 119 insertions(+), 84 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 436bf084685997..7d15d063a9ad6a 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -38,6 +38,7 @@ static inline PyObject* _PyLong_GetOne(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+1]; } PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); +PyObject *_PyLong_And(PyLongObject *left, PyLongObject *right); PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); PyObject *_PyLong_Subtract(PyLongObject *left, PyLongObject *right); diff --git a/Include/opcode.h b/Include/opcode.h index df45e7b5e5a30c..ece4670b8c5f97 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -117,67 +117,68 @@ extern "C" { #define BINARY_OP_ADD_INT 3 #define BINARY_OP_ADD_FLOAT 4 #define BINARY_OP_ADD_UNICODE 5 -#define BINARY_OP_INPLACE_ADD_UNICODE 6 -#define BINARY_OP_MULTIPLY_INT 7 -#define BINARY_OP_MULTIPLY_FLOAT 8 -#define BINARY_OP_SUBTRACT_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define COMPARE_OP_ADAPTIVE 16 -#define COMPARE_OP_FLOAT_JUMP 17 -#define COMPARE_OP_INT_JUMP 18 -#define COMPARE_OP_STR_JUMP 19 -#define BINARY_SUBSCR_ADAPTIVE 20 -#define BINARY_SUBSCR_GETITEM 21 -#define BINARY_SUBSCR_LIST_INT 22 -#define BINARY_SUBSCR_TUPLE_INT 23 -#define BINARY_SUBSCR_DICT 24 -#define STORE_SUBSCR_ADAPTIVE 26 -#define STORE_SUBSCR_LIST_INT 27 -#define STORE_SUBSCR_DICT 28 -#define CALL_ADAPTIVE 29 -#define CALL_BUILTIN_CLASS 34 -#define CALL_NO_KW_BUILTIN_O 36 -#define CALL_NO_KW_BUILTIN_FAST 37 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 -#define CALL_NO_KW_LEN 39 -#define CALL_NO_KW_ISINSTANCE 40 -#define CALL_PY_EXACT_ARGS 41 -#define CALL_PY_WITH_DEFAULTS 42 -#define CALL_NO_KW_LIST_APPEND 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 -#define CALL_NO_KW_STR_1 46 -#define CALL_NO_KW_TUPLE_1 47 -#define CALL_NO_KW_TYPE_1 48 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 55 -#define JUMP_ABSOLUTE_QUICK 56 -#define LOAD_ATTR_ADAPTIVE 57 -#define LOAD_ATTR_INSTANCE_VALUE 58 -#define LOAD_ATTR_WITH_HINT 59 -#define LOAD_ATTR_SLOT 62 -#define LOAD_ATTR_MODULE 63 -#define LOAD_GLOBAL_ADAPTIVE 64 -#define LOAD_GLOBAL_MODULE 65 -#define LOAD_GLOBAL_BUILTIN 66 -#define LOAD_METHOD_ADAPTIVE 67 -#define LOAD_METHOD_CACHED 72 -#define LOAD_METHOD_CLASS 76 -#define LOAD_METHOD_MODULE 77 -#define LOAD_METHOD_NO_DICT 78 -#define RESUME_QUICK 79 -#define STORE_ATTR_ADAPTIVE 80 -#define STORE_ATTR_INSTANCE_VALUE 81 -#define STORE_ATTR_SLOT 131 -#define STORE_ATTR_WITH_HINT 140 -#define UNPACK_SEQUENCE_ADAPTIVE 141 -#define UNPACK_SEQUENCE_LIST 143 -#define UNPACK_SEQUENCE_TUPLE 150 -#define UNPACK_SEQUENCE_TWO_TUPLE 153 -#define LOAD_FAST__LOAD_FAST 154 -#define STORE_FAST__LOAD_FAST 158 -#define LOAD_FAST__LOAD_CONST 159 -#define LOAD_CONST__LOAD_FAST 161 -#define STORE_FAST__STORE_FAST 166 +#define BINARY_OP_AND_INT 6 +#define BINARY_OP_INPLACE_ADD_UNICODE 7 +#define BINARY_OP_MULTIPLY_INT 8 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_SUBTRACT_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define COMPARE_OP_ADAPTIVE 17 +#define COMPARE_OP_FLOAT_JUMP 18 +#define COMPARE_OP_INT_JUMP 19 +#define COMPARE_OP_STR_JUMP 20 +#define BINARY_SUBSCR_ADAPTIVE 21 +#define BINARY_SUBSCR_GETITEM 22 +#define BINARY_SUBSCR_LIST_INT 23 +#define BINARY_SUBSCR_TUPLE_INT 24 +#define BINARY_SUBSCR_DICT 26 +#define STORE_SUBSCR_ADAPTIVE 27 +#define STORE_SUBSCR_LIST_INT 28 +#define STORE_SUBSCR_DICT 29 +#define CALL_ADAPTIVE 34 +#define CALL_BUILTIN_CLASS 36 +#define CALL_NO_KW_BUILTIN_O 37 +#define CALL_NO_KW_BUILTIN_FAST 38 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 39 +#define CALL_NO_KW_LEN 40 +#define CALL_NO_KW_ISINSTANCE 41 +#define CALL_PY_EXACT_ARGS 42 +#define CALL_PY_WITH_DEFAULTS 43 +#define CALL_NO_KW_LIST_APPEND 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 46 +#define CALL_NO_KW_STR_1 47 +#define CALL_NO_KW_TUPLE_1 48 +#define CALL_NO_KW_TYPE_1 55 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 56 +#define JUMP_ABSOLUTE_QUICK 57 +#define LOAD_ATTR_ADAPTIVE 58 +#define LOAD_ATTR_INSTANCE_VALUE 59 +#define LOAD_ATTR_WITH_HINT 62 +#define LOAD_ATTR_SLOT 63 +#define LOAD_ATTR_MODULE 64 +#define LOAD_GLOBAL_ADAPTIVE 65 +#define LOAD_GLOBAL_MODULE 66 +#define LOAD_GLOBAL_BUILTIN 67 +#define LOAD_METHOD_ADAPTIVE 72 +#define LOAD_METHOD_CACHED 76 +#define LOAD_METHOD_CLASS 77 +#define LOAD_METHOD_MODULE 78 +#define LOAD_METHOD_NO_DICT 79 +#define RESUME_QUICK 80 +#define STORE_ATTR_ADAPTIVE 81 +#define STORE_ATTR_INSTANCE_VALUE 131 +#define STORE_ATTR_SLOT 140 +#define STORE_ATTR_WITH_HINT 141 +#define UNPACK_SEQUENCE_ADAPTIVE 143 +#define UNPACK_SEQUENCE_LIST 150 +#define UNPACK_SEQUENCE_TUPLE 153 +#define UNPACK_SEQUENCE_TWO_TUPLE 154 +#define LOAD_FAST__LOAD_FAST 158 +#define STORE_FAST__LOAD_FAST 159 +#define LOAD_FAST__LOAD_CONST 161 +#define LOAD_CONST__LOAD_FAST 166 +#define STORE_FAST__STORE_FAST 169 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/opcode.py b/Lib/opcode.py index 8237aa7fa3d116..5b725e46239e6d 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -231,6 +231,7 @@ def jabs_op(name, op): "BINARY_OP_ADD_INT", "BINARY_OP_ADD_FLOAT", "BINARY_OP_ADD_UNICODE", + "BINARY_OP_AND_INT", "BINARY_OP_INPLACE_ADD_UNICODE", "BINARY_OP_MULTIPLY_INT", "BINARY_OP_MULTIPLY_FLOAT", diff --git a/Objects/longobject.c b/Objects/longobject.c index 3438906d842758..875d7154b26b1e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4994,6 +4994,12 @@ long_and(PyObject *a, PyObject *b) CHECK_BINOP(a, b); PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; + return _PyLong_And(x, y); +} + +PyObject * +_PyLong_And(PyLongObject *x, PyLongObject *y) +{ if (IS_MEDIUM_VALUE(x) && IS_MEDIUM_VALUE(y)) { return _PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); } diff --git a/Python/ceval.c b/Python/ceval.c index b900de53c8c928..4516136d7ff142 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2080,6 +2080,25 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NOTRACE_DISPATCH(); } + TARGET(BINARY_OP_AND_INT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); + PyObject *res = _PyLong_And((PyLongObject *)left, + (PyLongObject *)right); + SET_SECOND(res); + Py_DECREF(right); + Py_DECREF(left); + STACK_SHRINK(1); + if (res == NULL) { + goto error; + } + NOTRACE_DISPATCH(); + } + TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); PyObject *sub = POP(); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 1e137f93c72922..e8d91812036480 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -5,16 +5,17 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_AND_INT, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_INT_JUMP, @@ -23,18 +24,18 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_GETITEM, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR, + &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_STORE_SUBSCR_DICT, - &&TARGET_CALL_ADAPTIVE, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_ADAPTIVE, &&TARGET_PUSH_EXC_INFO, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -47,40 +48,39 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, + &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_JUMP_ABSOLUTE_QUICK, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_METHOD_CACHED, + &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_GET_AWAITABLE, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_METHOD_CACHED, &&TARGET_LOAD_METHOD_CLASS, &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_RESUME_QUICK, &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -130,7 +130,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, - &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_NO_INTERRUPT, @@ -139,36 +139,36 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_METHOD, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_PRECALL_FUNCTION, &&TARGET_PRECALL_METHOD, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, diff --git a/Python/specialize.c b/Python/specialize.c index 5fd7d0923a6983..1b83c7165dd031 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -557,7 +557,7 @@ initial_counter_value(void) { #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8 #define SPEC_FAIL_BINARY_OP_ADD_OTHER 9 #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10 -#define SPEC_FAIL_BINARY_OP_AND_INT 11 + #define SPEC_FAIL_BINARY_OP_AND_OTHER 12 #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13 #define SPEC_FAIL_BINARY_OP_LSHIFT 14 @@ -1780,9 +1780,6 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { return SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES; } - if (PyLong_CheckExact(lhs)) { - return SPEC_FAIL_BINARY_OP_AND_INT; - } return SPEC_FAIL_BINARY_OP_AND_OTHER; case NB_FLOOR_DIVIDE: case NB_INPLACE_FLOOR_DIVIDE: @@ -1865,6 +1862,16 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto success; } break; + case NB_AND: + case NB_INPLACE_AND: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + break; + } + if (PyLong_CheckExact(lhs)) { + *instr = _Py_MAKECODEUNIT(BINARY_OP_AND_INT, _Py_OPARG(*instr)); + goto success; + } + break; case NB_MULTIPLY: case NB_INPLACE_MULTIPLY: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { From 5c3c66b27c60194e20a179a3416276f86eeb66b9 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 17 Feb 2022 14:55:51 -0800 Subject: [PATCH 2/2] Specialize LSHIFT_INT and RSHIFT_INT --- Include/opcode.h | 122 ++++++++++++++++++++-------------------- Lib/opcode.py | 2 + Python/ceval.c | 56 +++++++++++++++--- Python/opcode_targets.h | 58 +++++++++---------- Python/specialize.c | 43 +++++++++++--- 5 files changed, 177 insertions(+), 104 deletions(-) diff --git a/Include/opcode.h b/Include/opcode.h index ece4670b8c5f97..ea5165c554d6b7 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -119,66 +119,68 @@ extern "C" { #define BINARY_OP_ADD_UNICODE 5 #define BINARY_OP_AND_INT 6 #define BINARY_OP_INPLACE_ADD_UNICODE 7 -#define BINARY_OP_MULTIPLY_INT 8 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_SUBTRACT_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define COMPARE_OP_ADAPTIVE 17 -#define COMPARE_OP_FLOAT_JUMP 18 -#define COMPARE_OP_INT_JUMP 19 -#define COMPARE_OP_STR_JUMP 20 -#define BINARY_SUBSCR_ADAPTIVE 21 -#define BINARY_SUBSCR_GETITEM 22 -#define BINARY_SUBSCR_LIST_INT 23 -#define BINARY_SUBSCR_TUPLE_INT 24 -#define BINARY_SUBSCR_DICT 26 -#define STORE_SUBSCR_ADAPTIVE 27 -#define STORE_SUBSCR_LIST_INT 28 -#define STORE_SUBSCR_DICT 29 -#define CALL_ADAPTIVE 34 -#define CALL_BUILTIN_CLASS 36 -#define CALL_NO_KW_BUILTIN_O 37 -#define CALL_NO_KW_BUILTIN_FAST 38 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_ISINSTANCE 41 -#define CALL_PY_EXACT_ARGS 42 -#define CALL_PY_WITH_DEFAULTS 43 -#define CALL_NO_KW_LIST_APPEND 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 46 -#define CALL_NO_KW_STR_1 47 -#define CALL_NO_KW_TUPLE_1 48 -#define CALL_NO_KW_TYPE_1 55 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 56 -#define JUMP_ABSOLUTE_QUICK 57 -#define LOAD_ATTR_ADAPTIVE 58 -#define LOAD_ATTR_INSTANCE_VALUE 59 -#define LOAD_ATTR_WITH_HINT 62 -#define LOAD_ATTR_SLOT 63 -#define LOAD_ATTR_MODULE 64 -#define LOAD_GLOBAL_ADAPTIVE 65 -#define LOAD_GLOBAL_MODULE 66 -#define LOAD_GLOBAL_BUILTIN 67 -#define LOAD_METHOD_ADAPTIVE 72 -#define LOAD_METHOD_CACHED 76 -#define LOAD_METHOD_CLASS 77 -#define LOAD_METHOD_MODULE 78 -#define LOAD_METHOD_NO_DICT 79 -#define RESUME_QUICK 80 -#define STORE_ATTR_ADAPTIVE 81 -#define STORE_ATTR_INSTANCE_VALUE 131 -#define STORE_ATTR_SLOT 140 -#define STORE_ATTR_WITH_HINT 141 -#define UNPACK_SEQUENCE_ADAPTIVE 143 -#define UNPACK_SEQUENCE_LIST 150 -#define UNPACK_SEQUENCE_TUPLE 153 -#define UNPACK_SEQUENCE_TWO_TUPLE 154 -#define LOAD_FAST__LOAD_FAST 158 -#define STORE_FAST__LOAD_FAST 159 -#define LOAD_FAST__LOAD_CONST 161 -#define LOAD_CONST__LOAD_FAST 166 -#define STORE_FAST__STORE_FAST 169 +#define BINARY_OP_LSHIFT_INT 8 +#define BINARY_OP_MULTIPLY_INT 13 +#define BINARY_OP_MULTIPLY_FLOAT 14 +#define BINARY_OP_RSHIFT_INT 16 +#define BINARY_OP_SUBTRACT_INT 17 +#define BINARY_OP_SUBTRACT_FLOAT 18 +#define COMPARE_OP_ADAPTIVE 19 +#define COMPARE_OP_FLOAT_JUMP 20 +#define COMPARE_OP_INT_JUMP 21 +#define COMPARE_OP_STR_JUMP 22 +#define BINARY_SUBSCR_ADAPTIVE 23 +#define BINARY_SUBSCR_GETITEM 24 +#define BINARY_SUBSCR_LIST_INT 26 +#define BINARY_SUBSCR_TUPLE_INT 27 +#define BINARY_SUBSCR_DICT 28 +#define STORE_SUBSCR_ADAPTIVE 29 +#define STORE_SUBSCR_LIST_INT 34 +#define STORE_SUBSCR_DICT 36 +#define CALL_ADAPTIVE 37 +#define CALL_BUILTIN_CLASS 38 +#define CALL_NO_KW_BUILTIN_O 39 +#define CALL_NO_KW_BUILTIN_FAST 40 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 41 +#define CALL_NO_KW_LEN 42 +#define CALL_NO_KW_ISINSTANCE 43 +#define CALL_PY_EXACT_ARGS 44 +#define CALL_PY_WITH_DEFAULTS 45 +#define CALL_NO_KW_LIST_APPEND 46 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 47 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 48 +#define CALL_NO_KW_STR_1 55 +#define CALL_NO_KW_TUPLE_1 56 +#define CALL_NO_KW_TYPE_1 57 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 58 +#define JUMP_ABSOLUTE_QUICK 59 +#define LOAD_ATTR_ADAPTIVE 62 +#define LOAD_ATTR_INSTANCE_VALUE 63 +#define LOAD_ATTR_WITH_HINT 64 +#define LOAD_ATTR_SLOT 65 +#define LOAD_ATTR_MODULE 66 +#define LOAD_GLOBAL_ADAPTIVE 67 +#define LOAD_GLOBAL_MODULE 72 +#define LOAD_GLOBAL_BUILTIN 76 +#define LOAD_METHOD_ADAPTIVE 77 +#define LOAD_METHOD_CACHED 78 +#define LOAD_METHOD_CLASS 79 +#define LOAD_METHOD_MODULE 80 +#define LOAD_METHOD_NO_DICT 81 +#define RESUME_QUICK 131 +#define STORE_ATTR_ADAPTIVE 140 +#define STORE_ATTR_INSTANCE_VALUE 141 +#define STORE_ATTR_SLOT 143 +#define STORE_ATTR_WITH_HINT 150 +#define UNPACK_SEQUENCE_ADAPTIVE 153 +#define UNPACK_SEQUENCE_LIST 154 +#define UNPACK_SEQUENCE_TUPLE 158 +#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define LOAD_FAST__LOAD_FAST 161 +#define STORE_FAST__LOAD_FAST 166 +#define LOAD_FAST__LOAD_CONST 169 +#define LOAD_CONST__LOAD_FAST 170 +#define STORE_FAST__STORE_FAST 173 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/opcode.py b/Lib/opcode.py index 5b725e46239e6d..aa65f2c8544476 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -233,8 +233,10 @@ def jabs_op(name, op): "BINARY_OP_ADD_UNICODE", "BINARY_OP_AND_INT", "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_LSHIFT_INT", "BINARY_OP_MULTIPLY_INT", "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_RSHIFT_INT", "BINARY_OP_SUBTRACT_INT", "BINARY_OP_SUBTRACT_FLOAT", "COMPARE_OP_ADAPTIVE", diff --git a/Python/ceval.c b/Python/ceval.c index 4516136d7ff142..1b9564bcb308cc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2082,16 +2082,56 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr TARGET(BINARY_OP_AND_INT) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + DEOPT_IF(!PyLong_CheckExact(lhs), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(rhs), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *res = _PyLong_And((PyLongObject *)left, - (PyLongObject *)right); + PyObject *res = _PyLong_And((PyLongObject *)lhs, + (PyLongObject *)rhs); SET_SECOND(res); - Py_DECREF(right); - Py_DECREF(left); + Py_DECREF(rhs); + Py_DECREF(lhs); + STACK_SHRINK(1); + if (res == NULL) { + goto error; + } + NOTRACE_DISPATCH(); + } + + TARGET(BINARY_OP_LSHIFT_INT) { + assert(cframe.use_tracing == 0); + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + DEOPT_IF(!PyLong_CheckExact(lhs), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(rhs), BINARY_OP); + DEOPT_IF(Py_SIZE(rhs) != 1, BINARY_OP); + STAT_INC(BINARY_OP, hit); + size_t r = ((PyLongObject *)rhs)->ob_digit[0]; + PyObject *res = _PyLong_Lshift(lhs, r); + SET_SECOND(res); + Py_DECREF(rhs); + Py_DECREF(lhs); + STACK_SHRINK(1); + if (res == NULL) { + goto error; + } + NOTRACE_DISPATCH(); + } + + TARGET(BINARY_OP_RSHIFT_INT) { + assert(cframe.use_tracing == 0); + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + DEOPT_IF(!PyLong_CheckExact(lhs), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(rhs), BINARY_OP); + DEOPT_IF(Py_SIZE(rhs) != 1, BINARY_OP); + STAT_INC(BINARY_OP, hit); + size_t r = ((PyLongObject *)rhs)->ob_digit[0]; + PyObject *res = _PyLong_Rshift(lhs, r); + SET_SECOND(res); + Py_DECREF(rhs); + Py_DECREF(lhs); STACK_SHRINK(1); if (res == NULL) { goto error; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index e8d91812036480..031165202e3e90 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -7,14 +7,16 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_BINARY_OP_AND_INT, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_LSHIFT_INT, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_RSHIFT_INT, + &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, @@ -22,19 +24,19 @@ static void *opcode_targets[256] = { &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_BINARY_SUBSCR_ADAPTIVE, &&TARGET_BINARY_SUBSCR_GETITEM, + &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_ADAPTIVE, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_ADAPTIVE, + &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_PUSH_EXC_INFO, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_CALL_ADAPTIVE, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_BUILTIN_FAST, @@ -46,41 +48,39 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_LIST_APPEND, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, + &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_JUMP_ABSOLUTE_QUICK, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_GET_AWAITABLE, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_LOAD_METHOD_CACHED, &&TARGET_LOAD_METHOD_CLASS, &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, - &&TARGET_RESUME_QUICK, - &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -130,7 +130,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_RESUME_QUICK, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_NO_INTERRUPT, @@ -139,40 +139,40 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_LOAD_METHOD, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_PRECALL_FUNCTION, &&TARGET_PRECALL_METHOD, - &&TARGET_STORE_FAST__STORE_FAST, - &&_unknown_opcode, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 1b83c7165dd031..077815e8d531a5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -557,23 +557,24 @@ initial_counter_value(void) { #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8 #define SPEC_FAIL_BINARY_OP_ADD_OTHER 9 #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10 - +#define SPEC_FAIL_BINARY_OP_LSHIFT_DIFFERENT_TYPES 11 #define SPEC_FAIL_BINARY_OP_AND_OTHER 12 #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13 -#define SPEC_FAIL_BINARY_OP_LSHIFT 14 +#define SPEC_FAIL_BINARY_OP_LSHIFT_OTHER 14 #define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 15 #define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 16 #define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 17 #define SPEC_FAIL_BINARY_OP_OR 18 #define SPEC_FAIL_BINARY_OP_POWER 19 #define SPEC_FAIL_BINARY_OP_REMAINDER 20 -#define SPEC_FAIL_BINARY_OP_RSHIFT 21 +#define SPEC_FAIL_BINARY_OP_RSHIFT_OTHER 22 #define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 22 #define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 23 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 25 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 26 #define SPEC_FAIL_BINARY_OP_XOR 27 +#define SPEC_FAIL_BINARY_OP_RSHIFT_DIFFERENT_TYPES 28 /* Calls */ #define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9 @@ -1766,7 +1767,7 @@ _Py_Specialize_CallNoKw( #ifdef Py_STATS static int -binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) +binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr) { switch (oparg) { case NB_ADD: @@ -1786,7 +1787,10 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE; case NB_LSHIFT: case NB_INPLACE_LSHIFT: - return SPEC_FAIL_BINARY_OP_LSHIFT; + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + return SPEC_FAIL_BINARY_OP_LSHIFT_DIFFERENT_TYPES; + } + return SPEC_FAIL_BINARY_OP_LSHIFT_OTHER; case NB_MATRIX_MULTIPLY: case NB_INPLACE_MATRIX_MULTIPLY: return SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY; @@ -1807,7 +1811,10 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_REMAINDER; case NB_RSHIFT: case NB_INPLACE_RSHIFT: - return SPEC_FAIL_BINARY_OP_RSHIFT; + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + return SPEC_FAIL_BINARY_OP_RSHIFT_DIFFERENT_TYPES; + } + return SPEC_FAIL_BINARY_OP_RSHIFT_OTHER; case NB_SUBTRACT: case NB_INPLACE_SUBTRACT: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { @@ -1872,6 +1879,17 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto success; } break; + case NB_LSHIFT: + case NB_INPLACE_LSHIFT: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + break; + } + if (PyLong_CheckExact(lhs) && Py_SIZE(rhs) == 1) { + *instr = _Py_MAKECODEUNIT(BINARY_OP_LSHIFT_INT, + _Py_OPARG(*instr)); + goto success; + } + break; case NB_MULTIPLY: case NB_INPLACE_MULTIPLY: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { @@ -1888,6 +1906,17 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto success; } break; + case NB_RSHIFT: + case NB_INPLACE_RSHIFT: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + break; + } + if (PyLong_CheckExact(lhs) && Py_SIZE(rhs) == 1) { + *instr = _Py_MAKECODEUNIT(BINARY_OP_RSHIFT_INT, + _Py_OPARG(*instr)); + goto success; + } + break; case NB_SUBTRACT: case NB_INPLACE_SUBTRACT: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { @@ -1916,7 +1945,7 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, #endif } SPECIALIZATION_FAIL( - BINARY_OP, binary_op_fail_kind(adaptive->original_oparg, lhs, rhs)); + BINARY_OP, binary_op_fail_kind(adaptive->original_oparg, lhs, rhs, instr)); STAT_INC(BINARY_OP, failure); cache_backoff(adaptive); return;