diff --git a/Lib/opcode.py b/Lib/opcode.py index 37e88e92df..ab6b765b4b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,9 +4,9 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"] +__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", + "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG"] # It's a chicken-and-egg I'm afraid: # We're imported before _opcode's made. @@ -23,6 +23,7 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') +hasarg = [] hasconst = [] hasname = [] hasjrel = [] @@ -30,13 +31,21 @@ haslocal = [] hascompare = [] hasfree = [] -hasnargs = [] # unused +hasexc = [] + +def is_pseudo(op): + return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE + +oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, + haslocal, hascompare, hasfree, hasexc] opmap = {} -opname = ['<%r>' % (op,) for op in range(256)] + +## pseudo opcodes (used in the compiler) mapped to the values +## they can become in the actual code. +_pseudo_ops = {} def def_op(name, op): - opname[op] = name opmap[name] = op def name_op(name, op): @@ -51,15 +60,23 @@ def jabs_op(name, op): def_op(name, op) hasjabs.append(op) +def pseudo_op(name, op, real_ops): + def_op(name, op) + _pseudo_ops[name] = real_ops + # add the pseudo opcode to the lists its targets are in + for oplist in oplists: + res = [opmap[rop] in oplist for rop in real_ops] + if any(res): + assert all(res) + oplist.append(op) + + # Instruction opcodes for compiled code # Blank lines correspond to available opcodes +def_op('CACHE', 0) def_op('POP_TOP', 1) -def_op('ROT_TWO', 2) -def_op('ROT_THREE', 3) -def_op('DUP_TOP', 4) -def_op('DUP_TOP_TWO', 5) -def_op('ROT_FOUR', 6) +def_op('PUSH_NULL', 2) def_op('NOP', 9) def_op('UNARY_POSITIVE', 10) @@ -67,68 +84,53 @@ def jabs_op(name, op): def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) -def_op('BINARY_MATRIX_MULTIPLY', 16) -def_op('INPLACE_MATRIX_MULTIPLY', 17) -def_op('BINARY_POWER', 19) -def_op('BINARY_MULTIPLY', 20) - -def_op('BINARY_MODULO', 22) -def_op('BINARY_ADD', 23) -def_op('BINARY_SUBTRACT', 24) def_op('BINARY_SUBSCR', 25) -def_op('BINARY_FLOOR_DIVIDE', 26) -def_op('BINARY_TRUE_DIVIDE', 27) -def_op('INPLACE_FLOOR_DIVIDE', 28) -def_op('INPLACE_TRUE_DIVIDE', 29) +def_op('BINARY_SLICE', 26) +def_op('STORE_SLICE', 27) + def_op('GET_LEN', 30) def_op('MATCH_MAPPING', 31) def_op('MATCH_SEQUENCE', 32) def_op('MATCH_KEYS', 33) -def_op('COPY_DICT_WITHOUT_KEYS', 34) + +def_op('PUSH_EXC_INFO', 35) +def_op('CHECK_EXC_MATCH', 36) +def_op('CHECK_EG_MATCH', 37) def_op('WITH_EXCEPT_START', 49) def_op('GET_AITER', 50) def_op('GET_ANEXT', 51) def_op('BEFORE_ASYNC_WITH', 52) - +def_op('BEFORE_WITH', 53) def_op('END_ASYNC_FOR', 54) -def_op('INPLACE_ADD', 55) -def_op('INPLACE_SUBTRACT', 56) -def_op('INPLACE_MULTIPLY', 57) +def_op('CLEANUP_THROW', 55) -def_op('INPLACE_MODULO', 59) def_op('STORE_SUBSCR', 60) def_op('DELETE_SUBSCR', 61) -def_op('BINARY_LSHIFT', 62) -def_op('BINARY_RSHIFT', 63) -def_op('BINARY_AND', 64) -def_op('BINARY_XOR', 65) -def_op('BINARY_OR', 66) -def_op('INPLACE_POWER', 67) + +# TODO: RUSTPYTHON +# Delete below def_op after updating coroutines.py +def_op('YIELD_FROM', 72) + def_op('GET_ITER', 68) def_op('GET_YIELD_FROM_ITER', 69) def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) -def_op('YIELD_FROM', 72) -def_op('GET_AWAITABLE', 73) + def_op('LOAD_ASSERTION_ERROR', 74) -def_op('INPLACE_LSHIFT', 75) -def_op('INPLACE_RSHIFT', 76) -def_op('INPLACE_AND', 77) -def_op('INPLACE_XOR', 78) -def_op('INPLACE_OR', 79) +def_op('RETURN_GENERATOR', 75) def_op('LIST_TO_TUPLE', 82) def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) def_op('SETUP_ANNOTATIONS', 85) -def_op('YIELD_VALUE', 86) -def_op('POP_BLOCK', 87) +def_op('ASYNC_GEN_WRAP', 87) +def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) -HAVE_ARGUMENT = 90 # Opcodes from here have an argument: +HAVE_ARGUMENT = 90 # real opcodes from here have an argument: name_op('STORE_NAME', 90) # Index in name list name_op('DELETE_NAME', 91) # "" @@ -139,7 +141,7 @@ def jabs_op(name, op): name_op('DELETE_ATTR', 96) # "" name_op('STORE_GLOBAL', 97) # "" name_op('DELETE_GLOBAL', 98) # "" -def_op('ROT_N', 99) +def_op('SWAP', 99) def_op('LOAD_CONST', 100) # Index in const list hasconst.append(100) name_op('LOAD_NAME', 101) # Index in name list @@ -152,45 +154,47 @@ def jabs_op(name, op): hascompare.append(107) name_op('IMPORT_NAME', 108) # Index in name list name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip -jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code -jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jabs_op('JUMP_ABSOLUTE', 113) # "" -jabs_op('POP_JUMP_IF_FALSE', 114) # "" -jabs_op('POP_JUMP_IF_TRUE', 115) # "" +jrel_op('JUMP_FORWARD', 110) # Number of words to skip +jrel_op('JUMP_IF_FALSE_OR_POP', 111) # Number of words to skip +jrel_op('JUMP_IF_TRUE_OR_POP', 112) # "" +jrel_op('POP_JUMP_IF_FALSE', 114) +jrel_op('POP_JUMP_IF_TRUE', 115) name_op('LOAD_GLOBAL', 116) # Index in name list def_op('IS_OP', 117) def_op('CONTAINS_OP', 118) def_op('RERAISE', 119) - -jabs_op('JUMP_IF_NOT_EXC_MATCH', 121) -jrel_op('SETUP_FINALLY', 122) # Distance to target address - -def_op('LOAD_FAST', 124) # Local variable number +def_op('COPY', 120) +def_op('BINARY_OP', 122) +jrel_op('SEND', 123) # Number of bytes to skip +def_op('LOAD_FAST', 124) # Local variable number, no null check haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) - -def_op('GEN_START', 129) # Kind of generator/coroutine +def_op('LOAD_FAST_CHECK', 127) # Local variable number +haslocal.append(127) +jrel_op('POP_JUMP_IF_NOT_NONE', 128) +jrel_op('POP_JUMP_IF_NONE', 129) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('CALL_FUNCTION', 131) # #args +def_op('GET_AWAITABLE', 131) def_op('MAKE_FUNCTION', 132) # Flags def_op('BUILD_SLICE', 133) # Number of items - -def_op('LOAD_CLOSURE', 135) +jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) +def_op('MAKE_CELL', 135) hasfree.append(135) -def_op('LOAD_DEREF', 136) +def_op('LOAD_CLOSURE', 136) hasfree.append(136) -def_op('STORE_DEREF', 137) +def_op('LOAD_DEREF', 137) hasfree.append(137) -def_op('DELETE_DEREF', 138) +def_op('STORE_DEREF', 138) hasfree.append(138) +def_op('DELETE_DEREF', 139) +hasfree.append(139) +jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs def_op('CALL_FUNCTION_EX', 142) # Flags -jrel_op('SETUP_WITH', 143) + def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 def_op('LIST_APPEND', 145) @@ -198,19 +202,247 @@ def jabs_op(name, op): def_op('MAP_ADD', 147) def_op('LOAD_CLASSDEREF', 148) hasfree.append(148) - +def_op('COPY_FREE_VARS', 149) +def_op('YIELD_VALUE', 150) +def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py def_op('MATCH_CLASS', 152) -jrel_op('SETUP_ASYNC_WITH', 154) def_op('FORMAT_VALUE', 155) def_op('BUILD_CONST_KEY_MAP', 156) def_op('BUILD_STRING', 157) -name_op('LOAD_METHOD', 160) -def_op('CALL_METHOD', 161) def_op('LIST_EXTEND', 162) def_op('SET_UPDATE', 163) def_op('DICT_MERGE', 164) def_op('DICT_UPDATE', 165) -del def_op, name_op, jrel_op, jabs_op +def_op('CALL', 171) +def_op('KW_NAMES', 172) +hasconst.append(172) + + +hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) + +MIN_PSEUDO_OPCODE = 256 + +pseudo_op('SETUP_FINALLY', 256, ['NOP']) +hasexc.append(256) +pseudo_op('SETUP_CLEANUP', 257, ['NOP']) +hasexc.append(257) +pseudo_op('SETUP_WITH', 258, ['NOP']) +hasexc.append(258) +pseudo_op('POP_BLOCK', 259, ['NOP']) + +pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) +pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) + +pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) + +MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 + +del def_op, name_op, jrel_op, jabs_op, pseudo_op + +opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +for op, i in opmap.items(): + opname[i] = op + + +_nb_ops = [ + ("NB_ADD", "+"), + ("NB_AND", "&"), + ("NB_FLOOR_DIVIDE", "//"), + ("NB_LSHIFT", "<<"), + ("NB_MATRIX_MULTIPLY", "@"), + ("NB_MULTIPLY", "*"), + ("NB_REMAINDER", "%"), + ("NB_OR", "|"), + ("NB_POWER", "**"), + ("NB_RSHIFT", ">>"), + ("NB_SUBTRACT", "-"), + ("NB_TRUE_DIVIDE", "/"), + ("NB_XOR", "^"), + ("NB_INPLACE_ADD", "+="), + ("NB_INPLACE_AND", "&="), + ("NB_INPLACE_FLOOR_DIVIDE", "//="), + ("NB_INPLACE_LSHIFT", "<<="), + ("NB_INPLACE_MATRIX_MULTIPLY", "@="), + ("NB_INPLACE_MULTIPLY", "*="), + ("NB_INPLACE_REMAINDER", "%="), + ("NB_INPLACE_OR", "|="), + ("NB_INPLACE_POWER", "**="), + ("NB_INPLACE_RSHIFT", ">>="), + ("NB_INPLACE_SUBTRACT", "-="), + ("NB_INPLACE_TRUE_DIVIDE", "/="), + ("NB_INPLACE_XOR", "^="), +] + +_specializations = { + "BINARY_OP": [ + "BINARY_OP_ADAPTIVE", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_ADD_INT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_SUBTRACT_INT", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_ADAPTIVE", + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_TUPLE_INT", + ], + "CALL": [ + "CALL_ADAPTIVE", + "CALL_PY_EXACT_ARGS", + "CALL_PY_WITH_DEFAULTS", + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_NO_KW_BUILTIN_FAST", + "CALL_NO_KW_BUILTIN_O", + "CALL_NO_KW_ISINSTANCE", + "CALL_NO_KW_LEN", + "CALL_NO_KW_LIST_APPEND", + "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", + "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + "CALL_NO_KW_METHOD_DESCRIPTOR_O", + "CALL_NO_KW_STR_1", + "CALL_NO_KW_TUPLE_1", + "CALL_NO_KW_TYPE_1", + ], + "COMPARE_OP": [ + "COMPARE_OP_ADAPTIVE", + "COMPARE_OP_FLOAT_JUMP", + "COMPARE_OP_INT_JUMP", + "COMPARE_OP_STR_JUMP", + ], + "EXTENDED_ARG": [ + "EXTENDED_ARG_QUICK", + ], + "FOR_ITER": [ + "FOR_ITER_ADAPTIVE", + "FOR_ITER_LIST", + "FOR_ITER_RANGE", + ], + "JUMP_BACKWARD": [ + "JUMP_BACKWARD_QUICK", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_ADAPTIVE", + # These potentially push [NULL, bound method] onto the stack. + "LOAD_ATTR_CLASS", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_WITH_HINT", + # These will always push [unbound method, self] onto the stack. + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_WITH_DICT", + "LOAD_ATTR_METHOD_WITH_VALUES", + ], + "LOAD_CONST": [ + "LOAD_CONST__LOAD_FAST", + ], + "LOAD_FAST": [ + "LOAD_FAST__LOAD_CONST", + "LOAD_FAST__LOAD_FAST", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_ADAPTIVE", + "LOAD_GLOBAL_BUILTIN", + "LOAD_GLOBAL_MODULE", + ], + "RESUME": [ + "RESUME_QUICK", + ], + "STORE_ATTR": [ + "STORE_ATTR_ADAPTIVE", + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "STORE_FAST": [ + "STORE_FAST__LOAD_FAST", + "STORE_FAST__STORE_FAST", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_ADAPTIVE", + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_ADAPTIVE", + "UNPACK_SEQUENCE_LIST", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_TWO_TUPLE", + ], +} +_specialized_instructions = [ + opcode for family in _specializations.values() for opcode in family +] +_specialization_stats = [ + "success", + "failure", + "hit", + "deferred", + "miss", + "deopt", +] + +_cache_format = { + "LOAD_GLOBAL": { + "counter": 1, + "index": 1, + "module_keys_version": 2, + "builtin_keys_version": 1, + }, + "BINARY_OP": { + "counter": 1, + }, + "UNPACK_SEQUENCE": { + "counter": 1, + }, + "COMPARE_OP": { + "counter": 1, + "mask": 1, + }, + "BINARY_SUBSCR": { + "counter": 1, + "type_version": 2, + "func_version": 1, + }, + "FOR_ITER": { + "counter": 1, + }, + "LOAD_ATTR": { + "counter": 1, + "version": 2, + "keys_version": 2, + "descr": 4, + }, + "STORE_ATTR": { + "counter": 1, + "version": 2, + "index": 1, + }, + "CALL": { + "counter": 1, + "func_version": 2, + "min_args": 1, + }, + "STORE_SUBSCR": { + "counter": 1, + }, +} + +_inline_cache_entries = [ + sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256) +] diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py new file mode 100644 index 0000000000..65630a2715 --- /dev/null +++ b/Lib/test/test_code.py @@ -0,0 +1,820 @@ +"""This module includes tests of the code object representation. + +>>> def f(x): +... def g(y): +... return x + y +... return g +... + +# TODO: RUSTPYTHON +>>> # dump(f.__code__) +name: f +argcount: 1 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x', 'g') +cellvars: ('x',) +freevars: () +nlocals: 2 +flags: 3 +consts: ('None', '') + +# TODO: RUSTPYTHON +>>> # dump(f(4).__code__) +name: g +argcount: 1 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('y',) +cellvars: () +freevars: ('x',) +nlocals: 1 +flags: 19 +consts: ('None',) + +>>> def h(x, y): +... a = x + y +... b = x - y +... c = a * b +... return c +... + +# TODO: RUSTPYTHON +>>> # dump(h.__code__) +name: h +argcount: 2 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x', 'y', 'a', 'b', 'c') +cellvars: () +freevars: () +nlocals: 5 +flags: 3 +consts: ('None',) + +>>> def attrs(obj): +... print(obj.attr1) +... print(obj.attr2) +... print(obj.attr3) + +# TODO: RUSTPYTHON +>>> # dump(attrs.__code__) +name: attrs +argcount: 1 +posonlyargcount: 0 +kwonlyargcount: 0 +names: ('print', 'attr1', 'attr2', 'attr3') +varnames: ('obj',) +cellvars: () +freevars: () +nlocals: 1 +flags: 3 +consts: ('None',) + +>>> def optimize_away(): +... 'doc string' +... 'not a docstring' +... 53 +... 0x53 + +# TODO: RUSTPYTHON +>>> # dump(optimize_away.__code__) +name: optimize_away +argcount: 0 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: () +cellvars: () +freevars: () +nlocals: 0 +flags: 3 +consts: ("'doc string'", 'None') + +>>> def keywordonly_args(a,b,*,k1): +... return a,b,k1 +... + +# TODO: RUSTPYTHON +>>> # dump(keywordonly_args.__code__) +name: keywordonly_args +argcount: 2 +posonlyargcount: 0 +kwonlyargcount: 1 +names: () +varnames: ('a', 'b', 'k1') +cellvars: () +freevars: () +nlocals: 3 +flags: 3 +consts: ('None',) + +>>> def posonly_args(a,b,/,c): +... return a,b,c +... + +# TODO: RUSTPYTHON +>>> # dump(posonly_args.__code__) +name: posonly_args +argcount: 3 +posonlyargcount: 2 +kwonlyargcount: 0 +names: () +varnames: ('a', 'b', 'c') +cellvars: () +freevars: () +nlocals: 3 +flags: 3 +consts: ('None',) + +""" + +import inspect +import sys +import threading +import doctest +import unittest +import textwrap +import weakref + +try: + import ctypes +except ImportError: + ctypes = None +from test.support import (cpython_only, + check_impl_detail, + gc_collect) +from test.support.script_helper import assert_python_ok +from test.support import threading_helper +from opcode import opmap +COPY_FREE_VARS = opmap['COPY_FREE_VARS'] + + +def consts(t): + """Yield a doctest-safe sequence of object reprs.""" + for elt in t: + r = repr(elt) + if r.startswith("" % elt.co_name + else: + yield r + +def dump(co): + """Print out a text representation of a code object.""" + for attr in ["name", "argcount", "posonlyargcount", + "kwonlyargcount", "names", "varnames",]: + # TODO: RUSTPYTHON + # "cellvars","freevars", "nlocals", "flags"]: + print("%s: %s" % (attr, getattr(co, "co_" + attr))) + print("consts:", tuple(consts(co.co_consts))) + +# Needed for test_closure_injection below +# Defined at global scope to avoid implicitly closing over __class__ +def external_getitem(self, i): + return f"Foreign getitem: {super().__getitem__(i)}" + +class CodeTest(unittest.TestCase): + + @cpython_only + def test_newempty(self): + import _testcapi + co = _testcapi.code_newempty("filename", "funcname", 15) + self.assertEqual(co.co_filename, "filename") + self.assertEqual(co.co_name, "funcname") + self.assertEqual(co.co_firstlineno, 15) + #Empty code object should raise, but not crash the VM + with self.assertRaises(Exception): + exec(co) + + @cpython_only + def test_closure_injection(self): + # From https://bugs.python.org/issue32176 + from types import FunctionType + + def create_closure(__class__): + return (lambda: __class__).__closure__ + + def new_code(c): + '''A new code object with a __class__ cell added to freevars''' + return c.replace(co_freevars=c.co_freevars + ('__class__',), co_code=bytes([COPY_FREE_VARS, 1])+c.co_code) + + def add_foreign_method(cls, name, f): + code = new_code(f.__code__) + assert not f.__closure__ + closure = create_closure(cls) + defaults = f.__defaults__ + setattr(cls, name, FunctionType(code, globals(), name, defaults, closure)) + + class List(list): + pass + + add_foreign_method(List, "__getitem__", external_getitem) + + # Ensure the closure injection actually worked + function = List.__getitem__ + class_ref = function.__closure__[0].cell_contents + self.assertIs(class_ref, List) + + # Ensure the zero-arg super() call in the injected method works + obj = List([1, 2, 3]) + self.assertEqual(obj[0], "Foreign getitem: 1") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_constructor(self): + def func(): pass + co = func.__code__ + CodeType = type(co) + + # test code constructor + CodeType(co.co_argcount, + co.co_posonlyargcount, + co.co_kwonlyargcount, + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_qualname, + co.co_firstlineno, + co.co_linetable, + co.co_exceptiontable, + co.co_freevars, + co.co_cellvars) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_qualname(self): + self.assertEqual( + CodeTest.test_qualname.__code__.co_qualname, + CodeTest.test_qualname.__qualname__ + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_replace(self): + def func(): + x = 1 + return x + code = func.__code__ + + # different co_name, co_varnames, co_consts + def func2(): + y = 2 + z = 3 + return y + code2 = func2.__code__ + + for attr, value in ( + ("co_argcount", 0), + ("co_posonlyargcount", 0), + ("co_kwonlyargcount", 0), + ("co_nlocals", 1), + ("co_stacksize", 0), + ("co_flags", code.co_flags | inspect.CO_COROUTINE), + ("co_firstlineno", 100), + ("co_code", code2.co_code), + ("co_consts", code2.co_consts), + ("co_names", ("myname",)), + ("co_varnames", ('spam',)), + ("co_freevars", ("freevar",)), + ("co_cellvars", ("cellvar",)), + ("co_filename", "newfilename"), + ("co_name", "newname"), + ("co_linetable", code2.co_linetable), + ): + with self.subTest(attr=attr, value=value): + new_code = code.replace(**{attr: value}) + self.assertEqual(getattr(new_code, attr), value) + + new_code = code.replace(co_varnames=code2.co_varnames, + co_nlocals=code2.co_nlocals) + self.assertEqual(new_code.co_varnames, code2.co_varnames) + self.assertEqual(new_code.co_nlocals, code2.co_nlocals) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_nlocals_mismatch(self): + def func(): + x = 1 + return x + co = func.__code__ + assert co.co_nlocals > 0; + + # First we try the constructor. + CodeType = type(co) + for diff in (-1, 1): + with self.assertRaises(ValueError): + CodeType(co.co_argcount, + co.co_posonlyargcount, + co.co_kwonlyargcount, + # This is the only change. + co.co_nlocals + diff, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_qualname, + co.co_firstlineno, + co.co_linetable, + co.co_exceptiontable, + co.co_freevars, + co.co_cellvars, + ) + # Then we try the replace method. + with self.assertRaises(ValueError): + co.replace(co_nlocals=co.co_nlocals - 1) + with self.assertRaises(ValueError): + co.replace(co_nlocals=co.co_nlocals + 1) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_shrinking_localsplus(self): + # Check that PyCode_NewWithPosOnlyArgs resizes both + # localsplusnames and localspluskinds, if an argument is a cell. + def func(arg): + return lambda: arg + code = func.__code__ + newcode = code.replace(co_name="func") # Should not raise SystemError + self.assertEqual(code, newcode) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_empty_linetable(self): + def func(): + pass + new_code = code = func.__code__.replace(co_linetable=b'') + self.assertEqual(list(new_code.co_lines()), []) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_invalid_bytecode(self): + def foo(): pass + foo.__code__ = co = foo.__code__.replace(co_code=b'\xee\x00d\x00S\x00') + + with self.assertRaises(SystemError) as se: + foo() + self.assertEqual( + f"{co.co_filename}:{co.co_firstlineno}: unknown opcode 238", + str(se.exception)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + # @requires_debug_ranges() + def test_co_positions_artificial_instructions(self): + import dis + + namespace = {} + exec(textwrap.dedent("""\ + try: + 1/0 + except Exception as e: + exc = e + """), namespace) + + exc = namespace['exc'] + traceback = exc.__traceback__ + code = traceback.tb_frame.f_code + + artificial_instructions = [] + for instr, positions in zip( + dis.get_instructions(code, show_caches=True), + code.co_positions(), + strict=True + ): + # If any of the positions is None, then all have to + # be None as well for the case above. There are still + # some places in the compiler, where the artificial instructions + # get assigned the first_lineno but they don't have other positions. + # There is no easy way of inferring them at that stage, so for now + # we don't support it. + self.assertIn(positions.count(None), [0, 3, 4]) + + if not any(positions): + artificial_instructions.append(instr) + + self.assertEqual( + [ + (instruction.opname, instruction.argval) + for instruction in artificial_instructions + ], + [ + ("PUSH_EXC_INFO", None), + ("LOAD_CONST", None), # artificial 'None' + ("STORE_NAME", "e"), # XX: we know the location for this + ("DELETE_NAME", "e"), + ("RERAISE", 1), + ("COPY", 3), + ("POP_EXCEPT", None), + ("RERAISE", 1) + ] + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_endline_and_columntable_none_when_no_debug_ranges(self): + # Make sure that if `-X no_debug_ranges` is used, there is + # minimal debug info + code = textwrap.dedent(""" + def f(): + pass + + positions = f.__code__.co_positions() + for line, end_line, column, end_column in positions: + assert line == end_line + assert column is None + assert end_column is None + """) + assert_python_ok('-X', 'no_debug_ranges', '-c', code) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_endline_and_columntable_none_when_no_debug_ranges_env(self): + # Same as above but using the environment variable opt out. + code = textwrap.dedent(""" + def f(): + pass + + positions = f.__code__.co_positions() + for line, end_line, column, end_column in positions: + assert line == end_line + assert column is None + assert end_column is None + """) + assert_python_ok('-c', code, PYTHONNODEBUGRANGES='1') + + # co_positions behavior when info is missing. + + # TODO: RUSTPYTHON + @unittest.expectedFailure + # @requires_debug_ranges() + def test_co_positions_empty_linetable(self): + def func(): + x = 1 + new_code = func.__code__.replace(co_linetable=b'') + positions = new_code.co_positions() + for line, end_line, column, end_column in positions: + self.assertIsNone(line) + self.assertEqual(end_line, new_code.co_firstlineno + 1) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_code_equality(self): + def f(): + try: + a() + except: + b() + else: + c() + finally: + d() + code_a = f.__code__ + code_b = code_a.replace(co_linetable=b"") + code_c = code_a.replace(co_exceptiontable=b"") + code_d = code_b.replace(co_exceptiontable=b"") + self.assertNotEqual(code_a, code_b) + self.assertNotEqual(code_a, code_c) + self.assertNotEqual(code_a, code_d) + self.assertNotEqual(code_b, code_c) + self.assertNotEqual(code_b, code_d) + self.assertNotEqual(code_c, code_d) + + +def isinterned(s): + return s is sys.intern(('_' + s + '_')[1:-1]) + +class CodeConstsTest(unittest.TestCase): + + def find_const(self, consts, value): + for v in consts: + if v == value: + return v + self.assertIn(value, consts) # raises an exception + self.fail('Should never be reached') + + def assertIsInterned(self, s): + if not isinterned(s): + self.fail('String %r is not interned' % (s,)) + + def assertIsNotInterned(self, s): + if isinterned(s): + self.fail('String %r is interned' % (s,)) + + @cpython_only + def test_interned_string(self): + co = compile('res = "str_value"', '?', 'exec') + v = self.find_const(co.co_consts, 'str_value') + self.assertIsInterned(v) + + @cpython_only + def test_interned_string_in_tuple(self): + co = compile('res = ("str_value",)', '?', 'exec') + v = self.find_const(co.co_consts, ('str_value',)) + self.assertIsInterned(v[0]) + + @cpython_only + def test_interned_string_in_frozenset(self): + co = compile('res = a in {"str_value"}', '?', 'exec') + v = self.find_const(co.co_consts, frozenset(('str_value',))) + self.assertIsInterned(tuple(v)[0]) + + @cpython_only + def test_interned_string_default(self): + def f(a='str_value'): + return a + self.assertIsInterned(f()) + + @cpython_only + def test_interned_string_with_null(self): + co = compile(r'res = "str\0value!"', '?', 'exec') + v = self.find_const(co.co_consts, 'str\0value!') + self.assertIsNotInterned(v) + + +class CodeWeakRefTest(unittest.TestCase): + + def test_basic(self): + # Create a code object in a clean environment so that we know we have + # the only reference to it left. + namespace = {} + exec("def f(): pass", globals(), namespace) + f = namespace["f"] + del namespace + + self.called = False + def callback(code): + self.called = True + + # f is now the last reference to the function, and through it, the code + # object. While we hold it, check that we can create a weakref and + # deref it. Then delete it, and check that the callback gets called and + # the reference dies. + coderef = weakref.ref(f.__code__, callback) + self.assertTrue(bool(coderef())) + del f + gc_collect() # For PyPy or other GCs. + self.assertFalse(bool(coderef())) + self.assertTrue(self.called) + +# Python implementation of location table parsing algorithm +def read(it): + return next(it) + +def read_varint(it): + b = read(it) + val = b & 63; + shift = 0; + while b & 64: + b = read(it) + shift += 6 + val |= (b&63) << shift + return val + +def read_signed_varint(it): + uval = read_varint(it) + if uval & 1: + return -(uval >> 1) + else: + return uval >> 1 + +def parse_location_table(code): + line = code.co_firstlineno + it = iter(code.co_linetable) + while True: + try: + first_byte = read(it) + except StopIteration: + return + code = (first_byte >> 3) & 15 + length = (first_byte & 7) + 1 + if code == 15: + yield (code, length, None, None, None, None) + elif code == 14: + line_delta = read_signed_varint(it) + line += line_delta + end_line = line + read_varint(it) + col = read_varint(it) + if col == 0: + col = None + else: + col -= 1 + end_col = read_varint(it) + if end_col == 0: + end_col = None + else: + end_col -= 1 + yield (code, length, line, end_line, col, end_col) + elif code == 13: # No column + line_delta = read_signed_varint(it) + line += line_delta + yield (code, length, line, line, None, None) + elif code in (10, 11, 12): # new line + line_delta = code - 10 + line += line_delta + column = read(it) + end_column = read(it) + yield (code, length, line, line, column, end_column) + else: + assert (0 <= code < 10) + second_byte = read(it) + column = code << 3 | (second_byte >> 4) + yield (code, length, line, line, column, column + (second_byte & 15)) + +def positions_from_location_table(code): + for _, length, line, end_line, col, end_col in parse_location_table(code): + for _ in range(length): + yield (line, end_line, col, end_col) + +def dedup(lst, prev=object()): + for item in lst: + if item != prev: + yield item + prev = item + +def lines_from_postions(positions): + return dedup(l for (l, _, _, _) in positions) + +def misshappen(): + """ + + + + + + """ + x = ( + + + 4 + + + + + y + + ) + y = ( + a + + + b + + + + d + ) + return q if ( + + x + + ) else p + +def bug93662(): + example_report_generation_message= ( + """ + """ + ).strip() + raise ValueError() + + +class CodeLocationTest(unittest.TestCase): + + def check_positions(self, func): + pos1 = list(func.__code__.co_positions()) + pos2 = list(positions_from_location_table(func.__code__)) + for l1, l2 in zip(pos1, pos2): + self.assertEqual(l1, l2) + self.assertEqual(len(pos1), len(pos2)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_positions(self): + self.check_positions(parse_location_table) + self.check_positions(misshappen) + self.check_positions(bug93662) + + def check_lines(self, func): + co = func.__code__ + lines1 = list(dedup(l for (_, _, l) in co.co_lines())) + lines2 = list(lines_from_postions(positions_from_location_table(co))) + for l1, l2 in zip(lines1, lines2): + self.assertEqual(l1, l2) + self.assertEqual(len(lines1), len(lines2)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_lines(self): + self.check_lines(parse_location_table) + self.check_lines(misshappen) + self.check_lines(bug93662) + + +if check_impl_detail(cpython=True) and ctypes is not None: + py = ctypes.pythonapi + freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) + + RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = py._PyCode_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = py._PyCode_GetExtra + GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp)) + GetExtra.restype = ctypes.c_int + + LAST_FREED = None + def myfree(ptr): + global LAST_FREED + LAST_FREED = ptr + + FREE_FUNC = freefunc(myfree) + FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + + class CoExtra(unittest.TestCase): + def get_func(self): + # Defining a function causes the containing function to have a + # reference to the code object. We need the code objects to go + # away, so we eval a lambda. + return eval('lambda:42') + + def test_get_non_code(self): + f = self.get_func() + + self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + + def test_bad_index(self): + f = self.get_func() + self.assertRaises(SystemError, SetExtra, f.__code__, + FREE_INDEX+100, ctypes.c_voidp(100)) + self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, + ctypes.c_voidp(100)), 0) + + def test_free_called(self): + # Verify that the provided free function gets invoked + # when the code object is cleaned up. + f = self.get_func() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) + del f + self.assertEqual(LAST_FREED, 100) + + def test_get_set(self): + # Test basic get/set round tripping. + f = self.get_func() + + extra = ctypes.c_voidp() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) + # reset should free... + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) + self.assertEqual(LAST_FREED, 200) + + extra = ctypes.c_voidp() + GetExtra(f.__code__, FREE_INDEX, extra) + self.assertEqual(extra.value, 300) + del f + + @threading_helper.requires_working_threading() + def test_free_different_thread(self): + # Freeing a code object on a different thread then + # where the co_extra was set should be safe. + f = self.get_func() + class ThreadTest(threading.Thread): + def __init__(self, f, test): + super().__init__() + self.f = f + self.test = test + def run(self): + del self.f + self.test.assertEqual(LAST_FREED, 500) + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) + tt = ThreadTest(f, self) + del f + tt.start() + tt.join() + self.assertEqual(LAST_FREED, 500) + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + + +if __name__ == "__main__": + unittest.main() diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 84cc91024d..e3e9bd9eee 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -215,6 +215,18 @@ impl PyRef { self.code.obj_name.to_owned() } + #[pygetset] + fn co_names(self, vm: &VirtualMachine) -> PyTupleRef { + let names = self + .code + .names + .deref() + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(); + vm.ctx.new_tuple(names) + } + #[pygetset] fn co_flags(self) -> u16 { self.code.flags.bits()