From 8bd5b3fd2b3337acc4615f194cd0067ed44ee65e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 23 Jun 2022 15:40:37 +0100 Subject: [PATCH 01/14] remove unused (and undocumented) hasnargs --- Lib/opcode.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index f515aaa9fc8ddc..a6c60a99082c56 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -6,7 +6,7 @@ __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs", "haslocal", "hascompare", "hasfree", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"] + "HAVE_ARGUMENT", "EXTENDED_ARG"] # It's a chicken-and-egg I'm afraid: # We're imported before _opcode's made. @@ -30,7 +30,6 @@ haslocal = [] hascompare = [] hasfree = [] -hasnargs = [] # unused opmap = {} opname = ['<%r>' % (op,) for op in range(256)] From 3c99934ca74103ff5ebcb130ae4fbd8d98083082 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 23 Jun 2022 15:39:43 +0100 Subject: [PATCH 02/14] add virtual opcodes to the opcode and dis modules --- Doc/library/dis.rst | 74 ++++++++++++++++++++++++++++-- Include/internal/pycore_opcode.h | 19 ++++++-- Include/opcode.h | 34 +++++++++++--- Lib/dis.py | 2 +- Lib/opcode.py | 61 +++++++++++++++++++++--- Lib/test/test__opcode.py | 9 ++-- Lib/test/test_dis.py | 4 +- Modules/_opcode.c | 7 +-- Python/compile.c | 44 ++++-------------- Python/makeopcodetargets.py | 3 +- Tools/scripts/generate_opcode_h.py | 50 ++++++++++++++------ 11 files changed, 222 insertions(+), 85 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 10154d8b089290..549cbde6c8b4ba 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1330,15 +1330,67 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: HAVE_ARGUMENT +.. opcode:: _HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't use their argument and those that do - (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + non-virtual opcodes which don't use their argument and those that do + (``< _HAVE_ARGUMENT`` and ``>= _HAVE_ARGUMENT``, respectively). + + See also the :data:`hasarg` collection, which contains virtual as well + as non-virtual opcodes that use their argument. .. versionchanged:: 3.6 - Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` - ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + Now every instruction has an argument, but opcodes ``< _HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= _HAVE_ARGUMENT`` had an argument. + + .. versionchanged:: 3.12 + This field was renamed from ``HAVE_ARGUMENT`` to ``_HAVE_ARGUMENT``, + and its use is now discouraged. Check membership in :data:`hasarg` instead. + + +**Virtual opcodes** + +These opcodes do not appear in python bytecode, they are used by the compiler +but are replaced by real opcodes or removed before bytecode is generated. + +.. opcode:: SETUP_FINALLY + + Set up an exception handler for the following code block. If an exception + occurs, the value stack level is restored to its current state and control + is transferred to the exception handler. + + +.. opcode:: SETUP_CLEANUP + + Like ``SETUP_FINALLY``, but in case of exception also pushes the last + instruction to the stack so that ``RERAISE`` can restore it. + + +.. opcode:: SETUP_WITH + + Set up an exception handler for a :keyword:`with` or :keyword:`async with` + block. Pushes the last executed instruction to the stack like ``SETUP_CLEANUP``. + + +.. opcode:: POP_BLOCK + + Marks the end of the code block associated with the last ``SETUP_FINALLY``, + ``SETUP_CLEANUP`` or ``SETUP_WITH``. + +.. opcode:: JUMP +.. opcode:: JUMP_NO_INTERRUPT +.. opcode:: POP_JUMP_IF_FALSE +.. opcode:: POP_JUMP_IF_TRUE +.. opcode:: POP_JUMP_IF_NONE +.. opcode:: POP_JUMP_IF_NOT_NONE + + Undirected relative jump instructions which are replaced by their + directed (forward/backward) counterparts by the assembler. + +.. opcode:: LOAD_METHOD + + Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode + with a flag set in the arg. .. _opcode_collections: @@ -1349,6 +1401,9 @@ Opcode collections These collections are provided for automatic introspection of bytecode instructions: + .. versionchanged:: 3.12 + The collections now contain virtual opcodes as well. + .. data:: opname Sequence of operation names, indexable using the bytecode. @@ -1364,6 +1419,11 @@ instructions: Sequence of all compare operation names. +.. data:: hasarg + + Sequence of bytecodes that use their argument. + + .. data:: hasconst Sequence of bytecodes that access a constant. @@ -1400,3 +1460,7 @@ instructions: .. data:: hascompare Sequence of bytecodes of Boolean operations. + +.. data:: hasexc + + Sequence of bytecodes that set an exception handler. diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index b9195f5c3299dc..ded6b8acf9c767 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -19,7 +19,7 @@ extern const uint8_t _PyOpcode_Deopt[256]; extern const uint8_t _PyOpcode_Original[256]; #ifdef NEED_OPCODE_TABLES -static const uint32_t _PyOpcode_RelativeJump[8] = { +static const uint32_t _PyOpcode_RelativeJump[9] = { 0U, 0U, 536870912U, @@ -28,8 +28,9 @@ static const uint32_t _PyOpcode_RelativeJump[8] = { 122880U, 0U, 0U, + 1008U, }; -static const uint32_t _PyOpcode_Jump[8] = { +static const uint32_t _PyOpcode_Jump[9] = { 0U, 0U, 536870912U, @@ -38,6 +39,7 @@ static const uint32_t _PyOpcode_Jump[8] = { 122880U, 0U, 0U, + 1008U, }; const uint8_t _PyOpcode_Caches[256] = { @@ -423,7 +425,7 @@ const uint8_t _PyOpcode_Original[256] = { #endif // NEED_OPCODE_TABLES #ifdef Py_DEBUG -static const char *const _PyOpcode_OpName[256] = { +static const char *const _PyOpcode_OpName[267] = { [CACHE] = "CACHE", [POP_TOP] = "POP_TOP", [PUSH_NULL] = "PUSH_NULL", @@ -680,6 +682,17 @@ static const char *const _PyOpcode_OpName[256] = { [253] = "<253>", [254] = "<254>", [DO_TRACING] = "DO_TRACING", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_WITH] = "SETUP_WITH", + [POP_BLOCK] = "POP_BLOCK", + [JUMP] = "JUMP", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", + [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", + [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", + [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", + [LOAD_METHOD] = "LOAD_METHOD", }; #endif diff --git a/Include/opcode.h b/Include/opcode.h index 7a22d5257a669b..f77f23b8e320dc 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -45,7 +45,7 @@ extern "C" { #define ASYNC_GEN_WRAP 87 #define PREP_RERAISE_STAR 88 #define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 +#define _HAVE_ARGUMENT 90 #define STORE_NAME 90 #define DELETE_NAME 91 #define UNPACK_SEQUENCE 92 @@ -118,6 +118,19 @@ extern "C" { #define POP_JUMP_BACKWARD_IF_NONE 174 #define POP_JUMP_BACKWARD_IF_FALSE 175 #define POP_JUMP_BACKWARD_IF_TRUE 176 +#define MIN_VIRTUAL_OPCODE 256 +#define SETUP_FINALLY 256 +#define SETUP_CLEANUP 257 +#define SETUP_WITH 258 +#define POP_BLOCK 259 +#define JUMP 260 +#define JUMP_NO_INTERRUPT 261 +#define POP_JUMP_IF_FALSE 262 +#define POP_JUMP_IF_TRUE 263 +#define POP_JUMP_IF_NONE 264 +#define POP_JUMP_IF_NOT_NONE 265 +#define LOAD_METHOD 266 +#define MAX_VIRTUAL_OPCODE 266 #define BINARY_OP_ADAPTIVE 3 #define BINARY_OP_ADD_FLOAT 4 #define BINARY_OP_ADD_INT 5 @@ -192,9 +205,19 @@ extern "C" { #define UNPACK_SEQUENCE_TWO_TUPLE 180 #define DO_TRACING 255 +#define HAS_ARG(op) ((((op) >= _HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\ + || ((op) == JUMP) \ + || ((op) == JUMP_NO_INTERRUPT) \ + || ((op) == POP_JUMP_IF_FALSE) \ + || ((op) == POP_JUMP_IF_TRUE) \ + || ((op) == POP_JUMP_IF_NONE) \ + || ((op) == POP_JUMP_IF_NOT_NONE) \ + || ((op) == LOAD_METHOD) \ + ) + #define HAS_CONST(op) (false\ - || ((op) == 100) \ - || ((op) == 172) \ + || ((op) == LOAD_CONST) \ + || ((op) == KW_NAMES) \ ) #define NB_ADD 0 @@ -224,11 +247,8 @@ extern "C" { #define NB_INPLACE_TRUE_DIVIDE 24 #define NB_INPLACE_XOR 25 -#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) -/* Reserve some bytecodes for internal use in the compiler. - * The value of 240 is arbitrary. */ -#define IS_ARTIFICIAL(op) ((op) > 240) +#define IS_VIRTUAL_OPCODE(op) (((op) >= MIN_VIRTUAL_OPCODE) && ((op) <= MAX_VIRTUAL_OPCODE)) #ifdef __cplusplus } diff --git a/Lib/dis.py b/Lib/dis.py index bd87de97fc9ce6..d63f35a42b2467 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -610,7 +610,7 @@ def _unpack_opargs(code): op = code[i] deop = _deoptop(op) caches = _inline_cache_entries[deop] - if deop >= HAVE_ARGUMENT: + if deop in hasarg: arg = code[i+1] | extended_arg extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0 # The oparg is stored as a signed integer diff --git a/Lib/opcode.py b/Lib/opcode.py index a6c60a99082c56..c9b4d391ffccfc 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,9 +4,9 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] +__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", + "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", + "_HAVE_ARGUMENT", "EXTENDED_ARG"] # It's a chicken-and-egg I'm afraid: # We're imported before _opcode's made. @@ -23,6 +23,7 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') +hasarg = [] hasconst = [] hasname = [] hasjrel = [] @@ -30,12 +31,21 @@ haslocal = [] hascompare = [] hasfree = [] +hasexc = [] + +def is_virtual(op): + return op >= MIN_VIRTUAL_OPCODE and op <= MAX_VIRTUAL_OPCODE + +oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, + haslocal, hascompare, hasfree, hasexc] opmap = {} -opname = ['<%r>' % (op,) for op in range(256)] + +## virtual opcodes (used in the compiler) mapped to the values +## they can become in the actual code. +_virtual_ops = {} def def_op(name, op): - opname[op] = name opmap[name] = op def name_op(name, op): @@ -50,6 +60,17 @@ def jabs_op(name, op): def_op(name, op) hasjabs.append(op) +def virtual_op(name, op, real_ops): + def_op(name, op) + _virtual_ops[name] = real_ops + # add the virtual opcode to the lists its targets are in + for oplist in oplists: + res = [opmap[rop] in oplist for rop in real_ops] + if any(res): + assert all(res) + oplist.append(op) + + # Instruction opcodes for compiled code # Blank lines correspond to available opcodes @@ -102,7 +123,7 @@ def jabs_op(name, op): def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) -HAVE_ARGUMENT = 90 # Opcodes from here have an argument: +_HAVE_ARGUMENT = 90 # Non-virtual opcodes from here have an argument: name_op('STORE_NAME', 90) # Index in name list name_op('DELETE_NAME', 91) # "" @@ -197,8 +218,34 @@ def jabs_op(name, op): jrel_op('POP_JUMP_BACKWARD_IF_FALSE', 175) jrel_op('POP_JUMP_BACKWARD_IF_TRUE', 176) +hasarg.extend([op for op in opmap.values() if op >= _HAVE_ARGUMENT]) + +MIN_VIRTUAL_OPCODE = 256 + +virtual_op('SETUP_FINALLY', 256, ['NOP']) +hasexc.append(256) +virtual_op('SETUP_CLEANUP', 257, ['NOP']) +hasexc.append(257) +virtual_op('SETUP_WITH', 258, ['NOP']) +hasexc.append(258) +virtual_op('POP_BLOCK', 259, ['NOP']) + +virtual_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) +virtual_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) +virtual_op('POP_JUMP_IF_FALSE', 262, ['POP_JUMP_FORWARD_IF_FALSE', 'POP_JUMP_BACKWARD_IF_FALSE']) +virtual_op('POP_JUMP_IF_TRUE', 263, ['POP_JUMP_FORWARD_IF_TRUE', 'POP_JUMP_BACKWARD_IF_TRUE']) +virtual_op('POP_JUMP_IF_NONE', 264, ['POP_JUMP_FORWARD_IF_NONE', 'POP_JUMP_BACKWARD_IF_NONE']) +virtual_op('POP_JUMP_IF_NOT_NONE', 265, ['POP_JUMP_FORWARD_IF_NOT_NONE', 'POP_JUMP_BACKWARD_IF_NOT_NONE']) +virtual_op('LOAD_METHOD', 266, ['LOAD_ATTR']) + +MAX_VIRTUAL_OPCODE = MIN_VIRTUAL_OPCODE + len(_virtual_ops) - 1 + +del def_op, name_op, jrel_op, jabs_op, virtual_op + +opname = ['<%r>' % (op,) for op in range(MAX_VIRTUAL_OPCODE + 1)] +for op, i in opmap.items(): + opname[i] = op -del def_op, name_op, jrel_op, jabs_op _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 2a4c0d2eeb656a..f548e3647b31c7 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -18,9 +18,10 @@ def test_stack_effect(self): self.assertRaises(ValueError, stack_effect, dis.opmap['BUILD_SLICE']) self.assertRaises(ValueError, stack_effect, dis.opmap['POP_TOP'], 0) # All defined opcodes + has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): with self.subTest(opname=name): - if code < dis.HAVE_ARGUMENT: + if code not in has_arg: stack_effect(code) self.assertRaises(ValueError, stack_effect, code, 0) else: @@ -46,10 +47,12 @@ def test_stack_effect_jump(self): self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=True), 0) self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0) # All defined opcodes + has_arg = dis.hasarg + has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): with self.subTest(opname=name): - if code < dis.HAVE_ARGUMENT: + if code not in has_arg: common = stack_effect(code) jump = stack_effect(code, jump=True) nojump = stack_effect(code, jump=False) @@ -57,7 +60,7 @@ def test_stack_effect_jump(self): common = stack_effect(code, 0) jump = stack_effect(code, 0, jump=True) nojump = stack_effect(code, 0, jump=False) - if code in has_jump: + if code in has_jump or code in has_exc: self.assertEqual(common, max(jump, nojump)) else: self.assertEqual(jump, common) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 255425302f4280..1d34377b85b45b 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -879,7 +879,7 @@ def test_opname(self): def test_boundaries(self): self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG) - self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT) + self.assertEqual(dis.opmap["STORE_NAME"], dis._HAVE_ARGUMENT) def test_widths(self): long_opcodes = set(['POP_JUMP_FORWARD_IF_FALSE', @@ -897,7 +897,7 @@ def test_widths(self): continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH - if opcode < dis.HAVE_ARGUMENT: + if opcode in dis.hasarg: width += 1 + dis._OPARG_WIDTH self.assertLessEqual(len(opname), width) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 4812716c672718..99be977417743e 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -60,12 +60,7 @@ _opcode_stack_effect_impl(PyObject *module, int opcode, PyObject *oparg, "stack_effect: jump must be False, True or None"); return -1; } - if (IS_ARTIFICIAL(opcode)) { - effect = PY_INVALID_STACK_EFFECT; - } - else { - effect = PyCompile_OpcodeStackEffectWithJump(opcode, oparg_int, jump_int); - } + effect = PyCompile_OpcodeStackEffectWithJump(opcode, oparg_int, jump_int); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_SetString(PyExc_ValueError, "invalid opcode or oparg"); diff --git a/Python/compile.c b/Python/compile.c index 3946aac8c9943c..b11590743ec4f5 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -71,40 +71,14 @@ #define MAX_ALLOWED_STACK_USE (STACK_USE_GUIDELINE * 100) -/* Pseudo-instructions used in the compiler, - * but turned into NOPs or other instructions - * by the assembler. */ -#define SETUP_FINALLY -1 -#define SETUP_CLEANUP -2 -#define SETUP_WITH -3 -#define POP_BLOCK -4 -#define JUMP -5 -#define JUMP_NO_INTERRUPT -6 -#define POP_JUMP_IF_FALSE -7 -#define POP_JUMP_IF_TRUE -8 -#define POP_JUMP_IF_NONE -9 -#define POP_JUMP_IF_NOT_NONE -10 -#define LOAD_METHOD -11 - -#define MIN_VIRTUAL_OPCODE -11 -#define MAX_ALLOWED_OPCODE 254 +#define MAX_REAL_OPCODE 254 #define IS_WITHIN_OPCODE_RANGE(opcode) \ - ((opcode) >= MIN_VIRTUAL_OPCODE && (opcode) <= MAX_ALLOWED_OPCODE) - -#define IS_VIRTUAL_OPCODE(opcode) ((opcode) < 0) - -#define IS_VIRTUAL_JUMP_OPCODE(opcode) \ - ((opcode) == JUMP || \ - (opcode) == JUMP_NO_INTERRUPT || \ - (opcode) == POP_JUMP_IF_NONE || \ - (opcode) == POP_JUMP_IF_NOT_NONE || \ - (opcode) == POP_JUMP_IF_FALSE || \ - (opcode) == POP_JUMP_IF_TRUE) + (((opcode) >= 0 && (opcode) <= MAX_REAL_OPCODE) || \ + IS_VIRTUAL_OPCODE(opcode)) #define IS_JUMP_OPCODE(opcode) \ - (IS_VIRTUAL_JUMP_OPCODE(opcode) || \ - is_bit_set_in_table(_PyOpcode_Jump, opcode)) + is_bit_set_in_table(_PyOpcode_Jump, opcode) #define IS_BLOCK_PUSH_OPCODE(opcode) \ ((opcode) == SETUP_FINALLY || \ @@ -125,7 +99,6 @@ (opcode) == POP_JUMP_FORWARD_IF_FALSE || \ (opcode) == POP_JUMP_BACKWARD_IF_FALSE) - #define IS_BACKWARDS_JUMP_OPCODE(opcode) \ ((opcode) == JUMP_BACKWARD || \ (opcode) == JUMP_BACKWARD_NO_INTERRUPT || \ @@ -183,11 +156,11 @@ typedef struct exceptstack { static inline int is_bit_set_in_table(const uint32_t *table, int bitindex) { /* Is the relevant bit set in the relevant word? */ - /* 256 bits fit into 8 32-bits words. + /* 512 bits fit into 9 32-bits words. * Word is indexed by (bitindex>>ln(size of int in bits)). * Bit within word is the low bits of bitindex. */ - if (bitindex >= 0 && bitindex < 256) { + if (bitindex >= 0 && bitindex < 512) { uint32_t word = table[bitindex >> LOG_BITS_PER_INT]; return (word >> (bitindex & MASK_LOW_LOG_BITS)) & 1; } @@ -1270,7 +1243,7 @@ static int is_end_of_basic_block(struct instr *instr) { int opcode = instr->i_opcode; - return is_jump(instr) || IS_SCOPE_EXIT_OPCODE(opcode); + return IS_JUMP_OPCODE(opcode) || IS_SCOPE_EXIT_OPCODE(opcode); } static int @@ -1320,7 +1293,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, static int compiler_addop(struct compiler *c, int opcode, bool line) { - assert(!HAS_ARG(opcode) || IS_ARTIFICIAL(opcode)); + assert(!HAS_ARG(opcode)); if (compiler_use_new_implicit_block_if_needed(c) < 0) { return -1; } @@ -8942,7 +8915,6 @@ apply_static_swaps(basicblock *block, int i) static bool jump_thread(struct instr *inst, struct instr *target, int opcode) { - assert(!IS_VIRTUAL_OPCODE(opcode) || IS_VIRTUAL_JUMP_OPCODE(opcode)); assert(is_jump(inst)); assert(is_jump(target)); // bpo-45773: If inst->i_target == target->i_target, then nothing actually diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 3bf2e35ccb6dab..be046eb5cba73f 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -34,7 +34,8 @@ def write_contents(f): targets = ['_unknown_opcode'] * 256 targets[255] = "TARGET_DO_TRACING" for opname, op in opcode.opmap.items(): - targets[op] = "TARGET_%s" % opname + if not opcode.is_virtual(op): + targets[op] = "TARGET_%s" % opname next_op = 1 for opname in opcode._specialized_instructions: while targets[next_op] != '_unknown_opcode': diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index e1f4f01ae1de0c..bbb1da7a455703 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -20,11 +20,8 @@ """.lstrip() footer = """ -#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) -/* Reserve some bytecodes for internal use in the compiler. - * The value of 240 is arbitrary. */ -#define IS_ARTIFICIAL(op) ((op) > 240) +#define IS_VIRTUAL_OPCODE(op) (((op) >= MIN_VIRTUAL_OPCODE) && ((op) <= MAX_VIRTUAL_OPCODE)) #ifdef __cplusplus } @@ -63,8 +60,8 @@ def write_int_array_from_ops(name, ops, out): bits = 0 for op in ops: bits |= 1<>= 32 assert bits == 0 @@ -81,10 +78,19 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna exec(code, opcode) opmap = opcode['opmap'] opname = opcode['opname'] + hasarg = opcode['hasarg'] hasconst = opcode['hasconst'] hasjrel = opcode['hasjrel'] hasjabs = opcode['hasjabs'] - used = [ False ] * 256 + is_virtual = opcode['is_virtual'] + _virtual_ops = opcode['_virtual_ops'] + + _HAVE_ARGUMENT = opcode["_HAVE_ARGUMENT"] + MIN_VIRTUAL_OPCODE = opcode["MIN_VIRTUAL_OPCODE"] + MAX_VIRTUAL_OPCODE = opcode["MAX_VIRTUAL_OPCODE"] + + NUM_OPCODES = len(opname) + used = [ False ] * len(opname) next_op = 1 for name, op in opmap.items(): @@ -108,9 +114,17 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna for name in opname: if name in opmap: - fobj.write(DEFINE.format(name, opmap[name])) - if name == 'POP_EXCEPT': # Special entry for HAVE_ARGUMENT - fobj.write(DEFINE.format("HAVE_ARGUMENT", opcode["HAVE_ARGUMENT"])) + op = opmap[name] + if op == _HAVE_ARGUMENT: + fobj.write(DEFINE.format("_HAVE_ARGUMENT", _HAVE_ARGUMENT)) + if op == MIN_VIRTUAL_OPCODE: + fobj.write(DEFINE.format("MIN_VIRTUAL_OPCODE", MIN_VIRTUAL_OPCODE)) + + fobj.write(DEFINE.format(name, op)) + + if op == MAX_VIRTUAL_OPCODE: + fobj.write(DEFINE.format("MAX_VIRTUAL_OPCODE", MAX_VIRTUAL_OPCODE)) + for name, op in specialized_opmap.items(): fobj.write(DEFINE.format(name, op)) @@ -129,8 +143,9 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna iobj.write("};\n") deoptcodes = {} - for basic in opmap: - deoptcodes[basic] = basic + for basic, op in opmap.items(): + if not is_virtual(op): + deoptcodes[basic] = basic for basic, family in opcode["_specializations"].items(): for specialized in family: deoptcodes[specialized] = basic @@ -146,10 +161,17 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna iobj.write("};\n") iobj.write("#endif // NEED_OPCODE_TABLES\n") + fobj.write("\n") + fobj.write("#define HAS_ARG(op) ((((op) >= _HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\\") + for op in _virtual_ops: + if opmap[op] in hasarg: + fobj.write(f"\n || ((op) == {op}) \\") + fobj.write("\n )\n") + fobj.write("\n") fobj.write("#define HAS_CONST(op) (false\\") for op in hasconst: - fobj.write(f"\n || ((op) == {op}) \\") + fobj.write(f"\n || ((op) == {opname[op]}) \\") fobj.write("\n )\n") fobj.write("\n") @@ -158,7 +180,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna iobj.write("\n") iobj.write("#ifdef Py_DEBUG\n") - iobj.write("static const char *const _PyOpcode_OpName[256] = {\n") + iobj.write(f"static const char *const _PyOpcode_OpName[{NUM_OPCODES}] = {{\n") for op, name in enumerate(opname_including_specialized): if name[0] != "<": op = name From 14c5a1e5e2399861fdecdba36fcf9fab7279318c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 20:00:59 +0000 Subject: [PATCH 03/14] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst diff --git a/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst b/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst new file mode 100644 index 00000000000000..41030e1720764b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst @@ -0,0 +1 @@ +The :mod:`dis` module now has the opcodes for virtual instructions (those which are used by the compiler during code generation but then removed or replaced by real opcodes before the final bytecode is emitted). From de6470aa12ed7c3bd6b60f735d46720893eb71f5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 24 Jun 2022 21:07:16 +0100 Subject: [PATCH 04/14] add to whatsnew --- Doc/whatsnew/3.12.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8dde1350a7b166..b6618644581ba3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -90,6 +90,15 @@ New Modules Improved Modules ================ +dis +--- + +* Virtual opcodes are now exposed in :mod:`dis`. :data:`~dis.HAVE_ARGUMENT` + is still relevant to real opcodes, but it is not useful for virtual opcodes. + It is renamed to :data:`~dis._HAVE_ARGUMENT`, and you should use the + new :data:`~dis.hasarg` collection instead. + (Contributed by Irit Katriel in :gh:`94216`.) + os -- From 16a581619fcac86604db880d8646435f21ae4d0e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Jun 2022 16:23:09 +0100 Subject: [PATCH 05/14] versionadded for new collections --- Doc/library/dis.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 549cbde6c8b4ba..2755c842bfd150 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1423,6 +1423,8 @@ instructions: Sequence of bytecodes that use their argument. + .. versionadded:: 3.12 + .. data:: hasconst @@ -1464,3 +1466,5 @@ instructions: .. data:: hasexc Sequence of bytecodes that set an exception handler. + + .. versionadded:: 3.12 From 964b9039cf6353073d8db2f8402b572fbcec91ad Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Jun 2022 19:49:52 +0100 Subject: [PATCH 06/14] rename _HAVE_ARGUMENT back to HAVE_ARGUMENT --- Doc/library/dis.rst | 14 ++++++++------ Doc/whatsnew/3.12.rst | 4 ++-- Lib/test/test_dis.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e6d03e6b5a3bc9..8fb47835483bd1 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1346,22 +1346,24 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: _HAVE_ARGUMENT +.. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between non-virtual opcodes which don't use their argument and those that do - (``< _HAVE_ARGUMENT`` and ``>= _HAVE_ARGUMENT``, respectively). + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). See also the :data:`hasarg` collection, which contains virtual as well as non-virtual opcodes that use their argument. .. versionchanged:: 3.6 - Now every instruction has an argument, but opcodes ``< _HAVE_ARGUMENT`` - ignore it. Before, only opcodes ``>= _HAVE_ARGUMENT`` had an argument. + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - This field was renamed from ``HAVE_ARGUMENT`` to ``_HAVE_ARGUMENT``, - and its use is now discouraged. Check membership in :data:`hasarg` instead. + Virtual opcodes were added to the :mod:`dis` module, and for them it + is not true that comparison with ``HAVE_ARGUMENT`` indicates whether + they use their arg. Check membership in :data:`hasarg` instead if your + application uses virtual opcodes. **Virtual opcodes** diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 03b2680ecfae2d..6aa7e7728f71a2 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -95,8 +95,8 @@ dis * Virtual opcodes are now exposed in :mod:`dis`. :data:`~dis.HAVE_ARGUMENT` is still relevant to real opcodes, but it is not useful for virtual opcodes. - It is renamed to :data:`~dis._HAVE_ARGUMENT`, and you should use the - new :data:`~dis.hasarg` collection instead. + Use the new :data:`~dis.hasarg` collection instead if your application uses + virtual opcodes. (Contributed by Irit Katriel in :gh:`94216`.) os diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 1d34377b85b45b..07f1203ff06887 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -879,7 +879,7 @@ def test_opname(self): def test_boundaries(self): self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG) - self.assertEqual(dis.opmap["STORE_NAME"], dis._HAVE_ARGUMENT) + self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT) def test_widths(self): long_opcodes = set(['POP_JUMP_FORWARD_IF_FALSE', From 9887e02db289a46249a72b6748cd83d03a57d121 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Jun 2022 20:00:08 +0100 Subject: [PATCH 07/14] rename _HAVE_ARGUMENT back to HAVE_ARGUMENT (Lib/opcode.py) --- Lib/opcode.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index bc2f0527fa982f..1848e695794a4b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -6,7 +6,7 @@ __all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "_HAVE_ARGUMENT", "EXTENDED_ARG"] + "HAVE_ARGUMENT", "EXTENDED_ARG"] # It's a chicken-and-egg I'm afraid: # We're imported before _opcode's made. @@ -125,7 +125,7 @@ def virtual_op(name, op, real_ops): def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) -_HAVE_ARGUMENT = 90 # Non-virtual opcodes from here have an argument: +HAVE_ARGUMENT = 90 # Non-virtual opcodes from here have an argument: name_op('STORE_NAME', 90) # Index in name list name_op('DELETE_NAME', 91) # "" @@ -220,7 +220,7 @@ def virtual_op(name, op, real_ops): jrel_op('POP_JUMP_BACKWARD_IF_FALSE', 175) jrel_op('POP_JUMP_BACKWARD_IF_TRUE', 176) -hasarg.extend([op for op in opmap.values() if op >= _HAVE_ARGUMENT]) +hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_VIRTUAL_OPCODE = 256 From 80d08287b5c1920455a46358b89dc11c0d92379a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Jun 2022 20:07:20 +0100 Subject: [PATCH 08/14] rename _HAVE_ARGUMENT back to HAVE_ARGUMENT (Tools/) --- Tools/scripts/generate_opcode_h.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index bbb1da7a455703..cd1556ac9fa146 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -85,7 +85,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna is_virtual = opcode['is_virtual'] _virtual_ops = opcode['_virtual_ops'] - _HAVE_ARGUMENT = opcode["_HAVE_ARGUMENT"] + HAVE_ARGUMENT = opcode["HAVE_ARGUMENT"] MIN_VIRTUAL_OPCODE = opcode["MIN_VIRTUAL_OPCODE"] MAX_VIRTUAL_OPCODE = opcode["MAX_VIRTUAL_OPCODE"] @@ -115,8 +115,8 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna for name in opname: if name in opmap: op = opmap[name] - if op == _HAVE_ARGUMENT: - fobj.write(DEFINE.format("_HAVE_ARGUMENT", _HAVE_ARGUMENT)) + if op == HAVE_ARGUMENT: + fobj.write(DEFINE.format("HAVE_ARGUMENT", HAVE_ARGUMENT)) if op == MIN_VIRTUAL_OPCODE: fobj.write(DEFINE.format("MIN_VIRTUAL_OPCODE", MIN_VIRTUAL_OPCODE)) @@ -162,7 +162,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna iobj.write("#endif // NEED_OPCODE_TABLES\n") fobj.write("\n") - fobj.write("#define HAS_ARG(op) ((((op) >= _HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\\") + fobj.write("#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\\") for op in _virtual_ops: if opmap[op] in hasarg: fobj.write(f"\n || ((op) == {op}) \\") From 986fd5d1af4550d529777126cb217575730a3e1e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Jun 2022 20:29:01 +0100 Subject: [PATCH 09/14] rename _HAVE_ARGUMENT back to HAVE_ARGUMENT (regen-all) --- Include/opcode.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/opcode.h b/Include/opcode.h index 7e14fe762197cc..71d7d5e987c992 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -47,7 +47,7 @@ extern "C" { #define ASYNC_GEN_WRAP 87 #define PREP_RERAISE_STAR 88 #define POP_EXCEPT 89 -#define _HAVE_ARGUMENT 90 +#define HAVE_ARGUMENT 90 #define STORE_NAME 90 #define DELETE_NAME 91 #define UNPACK_SEQUENCE 92 @@ -207,7 +207,7 @@ extern "C" { #define UNPACK_SEQUENCE_TWO_TUPLE 182 #define DO_TRACING 255 -#define HAS_ARG(op) ((((op) >= _HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\ +#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\ || ((op) == JUMP) \ || ((op) == JUMP_NO_INTERRUPT) \ || ((op) == POP_JUMP_IF_FALSE) \ From e91a8f54853c05ddbdf6c7de2c77a069ffba13b7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 28 Jun 2022 00:45:29 +0100 Subject: [PATCH 10/14] applied Guido's comments --- Doc/library/dis.rst | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 8fb47835483bd1..03869d7ad43f6d 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1371,23 +1371,32 @@ iterations of the loop. These opcodes do not appear in python bytecode, they are used by the compiler but are replaced by real opcodes or removed before bytecode is generated. -.. opcode:: SETUP_FINALLY +.. opcode:: SETUP_FINALLY (target) Set up an exception handler for the following code block. If an exception occurs, the value stack level is restored to its current state and control - is transferred to the exception handler. + is transferred to the exception handler at ``target``. -.. opcode:: SETUP_CLEANUP +.. opcode:: SETUP_CLEANUP (target) Like ``SETUP_FINALLY``, but in case of exception also pushes the last - instruction to the stack so that ``RERAISE`` can restore it. + instruction (``lasti``) to the stack so that ``RERAISE`` can restore it. + If an exception occurs, the value stack level and the last instruction on + the frame are restored to their current state, and control is transferred + to the exception handler at ``target``. -.. opcode:: SETUP_WITH +.. opcode:: SETUP_WITH (target) Set up an exception handler for a :keyword:`with` or :keyword:`async with` - block. Pushes the last executed instruction to the stack like ``SETUP_CLEANUP``. + block. Pushes the last executed instruction (``lasti``) to the stack like + ``SETUP_CLEANUP``. If an exception occurs, the value stack level and the + last instruction on the frame are restored to their current state. The + the context manager's :meth:`~object.__exit__` or + :meth:`~object.__aexit__` (which was poped from the stack) is called, its + return value is pushed to the stack, and control is transferred to the + exception handler at ``target``. .. opcode:: POP_BLOCK @@ -1420,7 +1429,9 @@ These collections are provided for automatic introspection of bytecode instructions: .. versionchanged:: 3.12 - The collections now contain virtual opcodes as well. + The collections now contain virtual opcodes as well. These are + opcodes in the range between ``MIN_VIRTUAL_OPCODE`` and + ``MAX_VIRTUAL_OPCODE``. .. data:: opname From 1fa525527f8e66bca3d5b9ebd20dd15561f84cf0 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 30 Jun 2022 11:42:09 +0100 Subject: [PATCH 11/14] fix error in SETUP_WITH documentation --- Doc/library/dis.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 03869d7ad43f6d..4e065585e38fc9 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1389,14 +1389,13 @@ but are replaced by real opcodes or removed before bytecode is generated. .. opcode:: SETUP_WITH (target) - Set up an exception handler for a :keyword:`with` or :keyword:`async with` - block. Pushes the last executed instruction (``lasti``) to the stack like - ``SETUP_CLEANUP``. If an exception occurs, the value stack level and the - last instruction on the frame are restored to their current state. The - the context manager's :meth:`~object.__exit__` or - :meth:`~object.__aexit__` (which was poped from the stack) is called, its - return value is pushed to the stack, and control is transferred to the - exception handler at ``target``. + Like ``SETUP_FINALLY``, but in case of exception one more item is popped + from the stack before control is transferred to the exception handler at + ``target``. + + This variant is used in :keyword:`with` and :keyword:`async with` + constructs, which push the context manager's :meth:`~object.__enter__` + or :meth:`~object.__aenter__` to the stack. .. opcode:: POP_BLOCK From 01042673bf7eb94e607d2630decd704a1f3ba310 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 30 Jun 2022 12:04:50 +0100 Subject: [PATCH 12/14] remove mention of MAX_VIRTUAL_OPCODE from dis doc --- Doc/library/dis.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 4e065585e38fc9..0ebfb4ad71297e 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1429,8 +1429,7 @@ instructions: .. versionchanged:: 3.12 The collections now contain virtual opcodes as well. These are - opcodes in the range between ``MIN_VIRTUAL_OPCODE`` and - ``MAX_VIRTUAL_OPCODE``. + opcodes with values ``>= MIN_VIRTUAL_OPCODE``. .. data:: opname From 0cd9b73f6a0f1d1cab7ebda8157c700bce083df7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 30 Jun 2022 17:52:10 +0100 Subject: [PATCH 13/14] virtual --> pseudo --- Doc/library/dis.rst | 21 ++++----- Doc/whatsnew/3.12.rst | 10 ++-- Include/opcode.h | 8 ++-- Lib/opcode.py | 46 +++++++++---------- ...2-06-24-20-00-57.gh-issue-94216.hxnQPu.rst | 2 +- Python/compile.c | 6 +-- Python/makeopcodetargets.py | 2 +- Tools/scripts/generate_opcode_h.py | 24 +++++----- 8 files changed, 60 insertions(+), 59 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 0ebfb4ad71297e..055ae1e736538b 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1349,24 +1349,23 @@ iterations of the loop. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - non-virtual opcodes which don't use their argument and those that do - (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + opcodes in the range [0,255] which don't use their argument and those + that do (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). - See also the :data:`hasarg` collection, which contains virtual as well - as non-virtual opcodes that use their argument. + If your application uses pseudo instructions, use the :data:`hasarg` + collection instead. .. versionchanged:: 3.6 Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - Virtual opcodes were added to the :mod:`dis` module, and for them it - is not true that comparison with ``HAVE_ARGUMENT`` indicates whether - they use their arg. Check membership in :data:`hasarg` instead if your - application uses virtual opcodes. + Pseudo instructions were added to the :mod:`dis` module, and for them + it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether + they use their arg. -**Virtual opcodes** +**Pseudo-instructions** These opcodes do not appear in python bytecode, they are used by the compiler but are replaced by real opcodes or removed before bytecode is generated. @@ -1428,8 +1427,8 @@ These collections are provided for automatic introspection of bytecode instructions: .. versionchanged:: 3.12 - The collections now contain virtual opcodes as well. These are - opcodes with values ``>= MIN_VIRTUAL_OPCODE``. + The collections now contain pseudo instructions as well. These are + opcodes with values ``>= MIN_PSEUDO_OPCODE``. .. data:: opname diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 6aa7e7728f71a2..c3868f6078ff9f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -93,10 +93,12 @@ Improved Modules dis --- -* Virtual opcodes are now exposed in :mod:`dis`. :data:`~dis.HAVE_ARGUMENT` - is still relevant to real opcodes, but it is not useful for virtual opcodes. - Use the new :data:`~dis.hasarg` collection instead if your application uses - virtual opcodes. +* Pseudo instruction opcodes (which are used by the compiler but + do not appear in executable bytecode) are now exposed in the + :mod:`dis` module. + :data:`~dis.HAVE_ARGUMENT` is still relevant to real opcodes, + but it is not useful for pseudo instrcutions. Use the new + :data:`~dis.hasarg` collection instead. (Contributed by Irit Katriel in :gh:`94216`.) os diff --git a/Include/opcode.h b/Include/opcode.h index 71d7d5e987c992..f90ebb1d6e01cb 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -120,7 +120,7 @@ extern "C" { #define POP_JUMP_BACKWARD_IF_NONE 174 #define POP_JUMP_BACKWARD_IF_FALSE 175 #define POP_JUMP_BACKWARD_IF_TRUE 176 -#define MIN_VIRTUAL_OPCODE 256 +#define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 #define SETUP_WITH 258 @@ -132,7 +132,7 @@ extern "C" { #define POP_JUMP_IF_NONE 264 #define POP_JUMP_IF_NOT_NONE 265 #define LOAD_METHOD 266 -#define MAX_VIRTUAL_OPCODE 266 +#define MAX_PSEUDO_OPCODE 266 #define BINARY_OP_ADAPTIVE 3 #define BINARY_OP_ADD_FLOAT 4 #define BINARY_OP_ADD_INT 5 @@ -207,7 +207,7 @@ extern "C" { #define UNPACK_SEQUENCE_TWO_TUPLE 182 #define DO_TRACING 255 -#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\ +#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ || ((op) == JUMP_NO_INTERRUPT) \ || ((op) == POP_JUMP_IF_FALSE) \ @@ -250,7 +250,7 @@ extern "C" { #define NB_INPLACE_XOR 25 -#define IS_VIRTUAL_OPCODE(op) (((op) >= MIN_VIRTUAL_OPCODE) && ((op) <= MAX_VIRTUAL_OPCODE)) +#define IS_PSEUDO_OPCODE(op) (((op) >= MIN_PSEUDO_OPCODE) && ((op) <= MAX_PSEUDO_OPCODE)) #ifdef __cplusplus } diff --git a/Lib/opcode.py b/Lib/opcode.py index 1848e695794a4b..8a07fb75b2e97c 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -33,17 +33,17 @@ hasfree = [] hasexc = [] -def is_virtual(op): - return op >= MIN_VIRTUAL_OPCODE and op <= MAX_VIRTUAL_OPCODE +def is_pseudo(op): + return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, haslocal, hascompare, hasfree, hasexc] opmap = {} -## virtual opcodes (used in the compiler) mapped to the values +## pseudo opcodes (used in the compiler) mapped to the values ## they can become in the actual code. -_virtual_ops = {} +_pseudo_ops = {} def def_op(name, op): opmap[name] = op @@ -60,10 +60,10 @@ def jabs_op(name, op): def_op(name, op) hasjabs.append(op) -def virtual_op(name, op, real_ops): +def pseudo_op(name, op, real_ops): def_op(name, op) - _virtual_ops[name] = real_ops - # add the virtual opcode to the lists its targets are in + _pseudo_ops[name] = real_ops + # add the pseudo opcode to the lists its targets are in for oplist in oplists: res = [opmap[rop] in oplist for rop in real_ops] if any(res): @@ -125,7 +125,7 @@ def virtual_op(name, op, real_ops): def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) -HAVE_ARGUMENT = 90 # Non-virtual opcodes from here have an argument: +HAVE_ARGUMENT = 90 # real opcodes from here have an argument: name_op('STORE_NAME', 90) # Index in name list name_op('DELETE_NAME', 91) # "" @@ -222,29 +222,29 @@ def virtual_op(name, op, real_ops): hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) -MIN_VIRTUAL_OPCODE = 256 +MIN_PSEUDO_OPCODE = 256 -virtual_op('SETUP_FINALLY', 256, ['NOP']) +pseudo_op('SETUP_FINALLY', 256, ['NOP']) hasexc.append(256) -virtual_op('SETUP_CLEANUP', 257, ['NOP']) +pseudo_op('SETUP_CLEANUP', 257, ['NOP']) hasexc.append(257) -virtual_op('SETUP_WITH', 258, ['NOP']) +pseudo_op('SETUP_WITH', 258, ['NOP']) hasexc.append(258) -virtual_op('POP_BLOCK', 259, ['NOP']) +pseudo_op('POP_BLOCK', 259, ['NOP']) -virtual_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -virtual_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) -virtual_op('POP_JUMP_IF_FALSE', 262, ['POP_JUMP_FORWARD_IF_FALSE', 'POP_JUMP_BACKWARD_IF_FALSE']) -virtual_op('POP_JUMP_IF_TRUE', 263, ['POP_JUMP_FORWARD_IF_TRUE', 'POP_JUMP_BACKWARD_IF_TRUE']) -virtual_op('POP_JUMP_IF_NONE', 264, ['POP_JUMP_FORWARD_IF_NONE', 'POP_JUMP_BACKWARD_IF_NONE']) -virtual_op('POP_JUMP_IF_NOT_NONE', 265, ['POP_JUMP_FORWARD_IF_NOT_NONE', 'POP_JUMP_BACKWARD_IF_NOT_NONE']) -virtual_op('LOAD_METHOD', 266, ['LOAD_ATTR']) +pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) +pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) +pseudo_op('POP_JUMP_IF_FALSE', 262, ['POP_JUMP_FORWARD_IF_FALSE', 'POP_JUMP_BACKWARD_IF_FALSE']) +pseudo_op('POP_JUMP_IF_TRUE', 263, ['POP_JUMP_FORWARD_IF_TRUE', 'POP_JUMP_BACKWARD_IF_TRUE']) +pseudo_op('POP_JUMP_IF_NONE', 264, ['POP_JUMP_FORWARD_IF_NONE', 'POP_JUMP_BACKWARD_IF_NONE']) +pseudo_op('POP_JUMP_IF_NOT_NONE', 265, ['POP_JUMP_FORWARD_IF_NOT_NONE', 'POP_JUMP_BACKWARD_IF_NOT_NONE']) +pseudo_op('LOAD_METHOD', 266, ['LOAD_ATTR']) -MAX_VIRTUAL_OPCODE = MIN_VIRTUAL_OPCODE + len(_virtual_ops) - 1 +MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 -del def_op, name_op, jrel_op, jabs_op, virtual_op +del def_op, name_op, jrel_op, jabs_op, pseudo_op -opname = ['<%r>' % (op,) for op in range(MAX_VIRTUAL_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] for op, i in opmap.items(): opname[i] = op diff --git a/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst b/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst index 41030e1720764b..ae3c2e7e71f355 100644 --- a/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst +++ b/Misc/NEWS.d/next/Library/2022-06-24-20-00-57.gh-issue-94216.hxnQPu.rst @@ -1 +1 @@ -The :mod:`dis` module now has the opcodes for virtual instructions (those which are used by the compiler during code generation but then removed or replaced by real opcodes before the final bytecode is emitted). +The :mod:`dis` module now has the opcodes for pseudo instructions (those which are used by the compiler during code generation but then removed or replaced by real opcodes before the final bytecode is emitted). diff --git a/Python/compile.c b/Python/compile.c index a2210df8f109b0..4330cb9686a63b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -75,7 +75,7 @@ #define IS_WITHIN_OPCODE_RANGE(opcode) \ (((opcode) >= 0 && (opcode) <= MAX_REAL_OPCODE) || \ - IS_VIRTUAL_OPCODE(opcode)) + IS_PSEUDO_OPCODE(opcode)) #define IS_JUMP_OPCODE(opcode) \ is_bit_set_in_table(_PyOpcode_Jump, opcode) @@ -191,7 +191,7 @@ static int instr_size(struct instr *instruction) { int opcode = instruction->i_opcode; - assert(!IS_VIRTUAL_OPCODE(opcode)); + assert(!IS_PSEUDO_OPCODE(opcode)); int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); int caches = _PyOpcode_Caches[opcode]; @@ -202,7 +202,7 @@ static void write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) { int opcode = instruction->i_opcode; - assert(!IS_VIRTUAL_OPCODE(opcode)); + assert(!IS_PSEUDO_OPCODE(opcode)); int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index be046eb5cba73f..33a4b4a76a1253 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -34,7 +34,7 @@ def write_contents(f): targets = ['_unknown_opcode'] * 256 targets[255] = "TARGET_DO_TRACING" for opname, op in opcode.opmap.items(): - if not opcode.is_virtual(op): + if not opcode.is_pseudo(op): targets[op] = "TARGET_%s" % opname next_op = 1 for opname in opcode._specialized_instructions: diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index cd1556ac9fa146..1d21af8cac2816 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -21,7 +21,7 @@ footer = """ -#define IS_VIRTUAL_OPCODE(op) (((op) >= MIN_VIRTUAL_OPCODE) && ((op) <= MAX_VIRTUAL_OPCODE)) +#define IS_PSEUDO_OPCODE(op) (((op) >= MIN_PSEUDO_OPCODE) && ((op) <= MAX_PSEUDO_OPCODE)) #ifdef __cplusplus } @@ -82,12 +82,12 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna hasconst = opcode['hasconst'] hasjrel = opcode['hasjrel'] hasjabs = opcode['hasjabs'] - is_virtual = opcode['is_virtual'] - _virtual_ops = opcode['_virtual_ops'] + is_pseudo = opcode['is_pseudo'] + _pseudo_ops = opcode['_pseudo_ops'] HAVE_ARGUMENT = opcode["HAVE_ARGUMENT"] - MIN_VIRTUAL_OPCODE = opcode["MIN_VIRTUAL_OPCODE"] - MAX_VIRTUAL_OPCODE = opcode["MAX_VIRTUAL_OPCODE"] + MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] + MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] NUM_OPCODES = len(opname) used = [ False ] * len(opname) @@ -117,13 +117,13 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna op = opmap[name] if op == HAVE_ARGUMENT: fobj.write(DEFINE.format("HAVE_ARGUMENT", HAVE_ARGUMENT)) - if op == MIN_VIRTUAL_OPCODE: - fobj.write(DEFINE.format("MIN_VIRTUAL_OPCODE", MIN_VIRTUAL_OPCODE)) + if op == MIN_PSEUDO_OPCODE: + fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) fobj.write(DEFINE.format(name, op)) - if op == MAX_VIRTUAL_OPCODE: - fobj.write(DEFINE.format("MAX_VIRTUAL_OPCODE", MAX_VIRTUAL_OPCODE)) + if op == MAX_PSEUDO_OPCODE: + fobj.write(DEFINE.format("MAX_PSEUDO_OPCODE", MAX_PSEUDO_OPCODE)) for name, op in specialized_opmap.items(): @@ -144,7 +144,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna deoptcodes = {} for basic, op in opmap.items(): - if not is_virtual(op): + if not is_pseudo(op): deoptcodes[basic] = basic for basic, family in opcode["_specializations"].items(): for specialized in family: @@ -162,8 +162,8 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna iobj.write("#endif // NEED_OPCODE_TABLES\n") fobj.write("\n") - fobj.write("#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_VIRTUAL_OPCODE(op)))\\") - for op in _virtual_ops: + fobj.write("#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\\") + for op in _pseudo_ops: if opmap[op] in hasarg: fobj.write(f"\n || ((op) == {op}) \\") fobj.write("\n )\n") From 709755ce8e12700fc18f59b55378732fa85a0766 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 1 Jul 2022 14:35:40 +0100 Subject: [PATCH 14/14] fix two errors in SETUP_WITH documentation --- Doc/library/dis.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 4fa23df8ed9cec..9173c1bcf5e7ba 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1390,13 +1390,13 @@ but are replaced by real opcodes or removed before bytecode is generated. .. opcode:: SETUP_WITH (target) - Like ``SETUP_FINALLY``, but in case of exception one more item is popped + Like ``SETUP_CLEANUP``, but in case of exception one more item is popped from the stack before control is transferred to the exception handler at ``target``. This variant is used in :keyword:`with` and :keyword:`async with` - constructs, which push the context manager's :meth:`~object.__enter__` - or :meth:`~object.__aenter__` to the stack. + constructs, which push the return value of the context manager's + :meth:`~object.__enter__` or :meth:`~object.__aenter__` to the stack. .. opcode:: POP_BLOCK