diff --git a/compiler/ast.py b/compiler/ast.py index e6f7627..6d1a23c 100644 --- a/compiler/ast.py +++ b/compiler/ast.py @@ -527,7 +527,7 @@ def getChildNodes(self): def __repr__(self): return "For(%s, %s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.body), repr(self.else_)) -class From(Node): +class ImportFrom(Node): def __init__(self, modname, names, level, lineno=None): self.modname = modname self.names = names @@ -541,7 +541,7 @@ def getChildNodes(self): return () def __repr__(self): - return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level)) + return "ImportFrom(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level)) class Function(Node): def __init__(self, decorators, name, argnames, defaults, flags, doc, code, lineno=None): diff --git a/compiler/future.py b/compiler/future.py index aa12c1d..44d7109 100644 --- a/compiler/future.py +++ b/compiler/future.py @@ -3,13 +3,14 @@ """ from __future__ import print_function -from compiler import ast, walk +import ast +from compiler import walk def is_future(stmt): """Return true if statement is a well-formed future statement""" - if not isinstance(stmt, ast.From): + if not isinstance(stmt, ast.ImportFrom): return 0 - if stmt.modname == "__future__": + if stmt.module == "__future__": return 1 else: return 0 @@ -24,14 +25,14 @@ def __init__(self): self.found = {} # set def visitModule(self, node): - stmt = node.node - for s in stmt.nodes: + for s in node.body: if not self.check_stmt(s): break def check_stmt(self, stmt): if is_future(stmt): - for name, asname in stmt.names: + for alias in stmt.names: + name = alias.name if name in self.features: self.found[name] = 1 else: @@ -48,10 +49,10 @@ def get_features(self): class BadFutureParser: """Check for invalid future statements""" - def visitFrom(self, node): + def visitImportFrom(self, node): if hasattr(node, 'valid_future'): return - if node.modname != "__future__": + if node.module != "__future__": return raise SyntaxError("invalid future statement " + repr(node)) diff --git a/compiler/pycodegen.py b/compiler/pycodegen.py index 7740ac7..854424e 100644 --- a/compiler/pycodegen.py +++ b/compiler/pycodegen.py @@ -6,7 +6,8 @@ import sys from io import StringIO -from compiler import ast, parse, walk, syntax +import ast +from compiler import parse, walk, syntax from compiler import pyassem, misc, future, symbols from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \ SC_FREE, SC_CELL @@ -155,29 +156,33 @@ def visitGlobal(self, node): for name in node.names: self.globals.add(name) - def visitFunction(self, node): + def visitFunctionDef(self, node): self.names.add(node.name) def visitLambda(self, node): pass def visitImport(self, node): - for name, alias in node.names: - self.names.add(alias or name) + for alias in node.names: + self.names.add(alias.asname or alias.name) - def visitFrom(self, node): - for name, alias in node.names: - self.names.add(alias or name) + def visitImportFrom(self, node): + for alias in node.names: + self.names.add(alias.asname or alias.name) - def visitClass(self, node): + def visitClassDef(self, node): self.names.add(node.name) def visitAssName(self, node): self.names.add(node.name) + def visitName(self, node): + if isinstance(node.ctx, ast.Store): + self.names.add(node.id) + def is_constant_false(node): - if isinstance(node, ast.Const): - if not node.value: + if isinstance(node, ast.Num): + if not node.n: return 1 return 0 @@ -351,9 +356,9 @@ def visitModule(self, node): if node.doc: self.emit('LOAD_CONST', node.doc) self.storeName('__doc__') - lnf = walk(node.node, self.NameFinder(), verbose=0) + lnf = walk(node.body, self.NameFinder(), verbose=0) self.locals.push(lnf.getLocals()) - self.visit(node.node) + self.visit(node.body) self.emit('LOAD_CONST', None) self.emit('RETURN_VALUE') @@ -361,10 +366,10 @@ def visitExpression(self, node): self.set_lineno(node) self.scopes = self.parseSymbols(node) self.scope = self.scopes[node] - self.visit(node.node) + self.visit(node.body) self.emit('RETURN_VALUE') - def visitFunction(self, node): + def visitFunctionDef(self, node): self._visitFuncOrLambda(node, isLambda=0) if node.doc: self.setDocstring(node.doc) @@ -374,28 +379,28 @@ def visitLambda(self, node): self._visitFuncOrLambda(node, isLambda=1) def _visitFuncOrLambda(self, node, isLambda=0): - if not isLambda and node.decorators: - for decorator in node.decorators.nodes: + if not isLambda and node.decorator_list: + for decorator in node.decorator_list: self.visit(decorator) - ndecorators = len(node.decorators.nodes) + ndecorators = len(node.decorator_list) else: ndecorators = 0 gen = self.FunctionGen(node, self.scopes, isLambda, self.class_name, self.get_module()) - walk(node.code, gen) + walk(node.body, gen) gen.finish() self.set_lineno(node) - for default in node.defaults: + for default in node.args.defaults: self.visit(default) - self._makeClosure(gen, len(node.defaults)) + self._makeClosure(gen, len(node.args.defaults)) for i in range(ndecorators): self.emit('CALL_FUNCTION', 1) - def visitClass(self, node): + def visitClassDef(self, node): gen = self.ClassGen(node, self.scopes, self.get_module()) - walk(node.code, gen) + walk(node.body, gen) gen.finish() self.set_lineno(node) self.emit('LOAD_CONST', node.name) @@ -412,23 +417,23 @@ def visitClass(self, node): # The next few implement control-flow statements def visitIf(self, node): + test = node.test + if is_constant_false(test): + # XXX will need to check generator stuff here + return end = self.newBlock() - numtests = len(node.tests) - for i in range(numtests): - test, suite = node.tests[i] - if is_constant_false(test): - # XXX will need to check generator stuff here - continue - self.set_lineno(test) - self.visit(test) - nextTest = self.newBlock() - self.emit('POP_JUMP_IF_FALSE', nextTest) - self.nextBlock() - self.visit(suite) + self.set_lineno(test) + self.visit(test) + orelse = None + if node.orelse: + orelse = self.newBlock() + self.emit('POP_JUMP_IF_FALSE', orelse or end) + self.nextBlock() + self.visit(node.body) + if node.orelse: self.emit('JUMP_FORWARD', end) - self.startBlock(nextTest) - if node.else_: - self.visit(node.else_) + self.startBlock(orelse) + self.visit(node.orelse) self.nextBlock(end) def visitWhile(self, node): @@ -454,8 +459,8 @@ def visitWhile(self, node): self.startBlock(else_) # or just the POPs if not else clause self.emit('POP_BLOCK') self.setups.pop() - if node.else_: - self.visit(node.else_) + if node.orelse: + self.visit(node.orelse) self.nextBlock(after) def visitFor(self, node): @@ -466,20 +471,20 @@ def visitFor(self, node): self.set_lineno(node) self.emit('SETUP_LOOP', after) - self.visit(node.list) + self.visit(node.iter) self.emit('GET_ITER') self.nextBlock(start) self.set_lineno(node, force=1) self.emit('FOR_ITER', anchor) - self.visit(node.assign) + self.visit(node.target) self.visit(node.body) self.emit('JUMP_ABSOLUTE', start) self.nextBlock(anchor) self.emit('POP_BLOCK') self.setups.pop() - if node.else_: - self.visit(node.else_) + if node.orelse: + self.visit(node.orelse) self.nextBlock(after) def visitBreak(self, node): @@ -518,32 +523,48 @@ def visitContinue(self, node): def visitTest(self, node, jump): end = self.newBlock() - for child in node.nodes[:-1]: + for child in node.values[:-1]: self.visit(child) self.emit(jump, end) self.nextBlock() - self.visit(node.nodes[-1]) + self.visit(node.values[-1]) self.nextBlock(end) - def visitAnd(self, node): - self.visitTest(node, 'JUMP_IF_FALSE_OR_POP') + _boolop_opcode = { + ast.And: "JUMP_IF_FALSE_OR_POP", + ast.Or: "JUMP_IF_TRUE_OR_POP", + } - def visitOr(self, node): - self.visitTest(node, 'JUMP_IF_TRUE_OR_POP') + def visitBoolOp(self, node): + opcode = self._boolop_opcode[type(node.op)] + self.visitTest(node, opcode) + + _cmp_opcode = { + ast.Eq: "==", + ast.NotEq: "!=", + ast.Lt: "<", + ast.LtE: "<=", + ast.Gt: ">", + ast.GtE: ">=", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in", + } def visitIfExp(self, node): endblock = self.newBlock() elseblock = self.newBlock() self.visit(node.test) self.emit('POP_JUMP_IF_FALSE', elseblock) - self.visit(node.then) + self.visit(node.body) self.emit('JUMP_FORWARD', endblock) self.nextBlock(elseblock) - self.visit(node.else_) + self.visit(node.orelse) self.nextBlock(endblock) def visitCompare(self, node): - self.visit(node.expr) + self.visit(node.left) cleanup = self.newBlock() for op, code in node.ops[:-1]: self.visit(code) @@ -554,9 +575,10 @@ def visitCompare(self, node): self.nextBlock() # now do the last comparison if node.ops: - op, code = node.ops[-1] + op = node.ops[-1] + code = node.comparators[-1] self.visit(code) - self.emit('COMPARE_OP', op) + self.emit('COMPARE_OP', self._cmp_opcode[type(op)]) if len(node.ops) > 1: end = self.newBlock() self.emit('JUMP_FORWARD', end) @@ -659,15 +681,27 @@ def visitListCompIf(self, node, branch): self.newBlock() def _makeClosure(self, gen, args): + # Construct qualname prefix + prefix = "" + parent = gen.scope.parent + while not isinstance(parent, symbols.ModuleScope): + if isinstance(parent, symbols.FunctionScope): + prefix = parent.name + ".." + prefix + else: + prefix = parent.name + "." + prefix + parent = parent.parent + frees = gen.scope.get_free_vars() if frees: for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('BUILD_TUPLE', len(frees)) self.emit('LOAD_CONST', gen) + self.emit('LOAD_CONST', prefix + gen.name) # py3 qualname self.emit('MAKE_CLOSURE', args) else: self.emit('LOAD_CONST', gen) + self.emit('LOAD_CONST', prefix + gen.name) # py3 qualname self.emit('MAKE_FUNCTION', args) def visitGenExpr(self, node): @@ -775,11 +809,26 @@ def visitRaise(self, node): n = n + 1 self.emit('RAISE_VARARGS', n) + def visitTry(self, node): + if node.finalbody and not node.handlers: + self.visitTryFinally(node) + return + if not node.finalbody and node.handlers: + self.visitTryExcept(node) + return + + # Split into 2 statements, try-except wrapped with try-finally + try_ex = ast.Try(body=node.body, handlers=node.handlers, orelse=node.orelse, finalbody=[]) + node.body = try_ex + node.handlers = [] + node.orelse = [] + self.visitTryFinally(node) + def visitTryExcept(self, node): body = self.newBlock() handlers = self.newBlock() end = self.newBlock() - if node.else_: + if node.orelse: lElse = self.newBlock() else: lElse = end @@ -795,7 +844,10 @@ def visitTryExcept(self, node): last = len(node.handlers) - 1 for i in range(len(node.handlers)): - expr, target, body = node.handlers[i] + handler = node.handlers[i] + expr = handler.type + target = handler.name + body = handler.body self.set_lineno(expr) if expr: self.emit('DUP_TOP') @@ -806,23 +858,36 @@ def visitTryExcept(self, node): self.nextBlock() self.emit('POP_TOP') if target: - self.visit(target) + self.visit(ast.Name(id=target, ctx=ast.Store())) else: self.emit('POP_TOP') self.emit('POP_TOP') - self.visit(body) + + if target: + protected = ast.Try( + body=body, handlers=[], orelse=[], + finalbody=[ + ast.Assign(targets=[ast.Name(id=target, ctx=ast.Store())], value=ast.NameConstant(value=None)), + ast.Delete(ast.Name(id=target, ctx=ast.Del())), + ] + ) + self.visitTryFinally(protected, except_protect=True) + else: + self.visit(body) + self.emit('POP_EXCEPT') + self.emit('JUMP_FORWARD', end) if expr: self.nextBlock(next) else: self.nextBlock() self.emit('END_FINALLY') - if node.else_: + if node.orelse: self.nextBlock(lElse) - self.visit(node.else_) + self.visit(node.orelse) self.nextBlock(end) - def visitTryFinally(self, node): + def visitTryFinally(self, node, except_protect=False): body = self.newBlock() final = self.newBlock() self.set_lineno(node) @@ -832,10 +897,12 @@ def visitTryFinally(self, node): self.visit(node.body) self.emit('POP_BLOCK') self.setups.pop() + if except_protect: + self.emit('POP_EXCEPT') self.emit('LOAD_CONST', None) self.nextBlock(final) self.setups.push((END_FINALLY, final)) - self.visit(node.final) + self.visit(node.finalbody) self.emit('END_FINALLY') self.setups.pop() @@ -882,6 +949,23 @@ def visitDiscard(self, node): self.visit(node.expr) self.emit('POP_TOP') + def visitExpr(self, node): + self.set_lineno(node) + self.visit(node.value) + self.emit('POP_TOP') + + def visitNum(self, node): + self.emit('LOAD_CONST', node.n) + + def visitStr(self, node): + self.emit('LOAD_CONST', node.s) + + def visitBytes(self, node): + self.emit('LOAD_CONST', node.s) + + def visitNameConstant(self, node): + self.emit('LOAD_CONST', node.value) + def visitConst(self, node): self.emit('LOAD_CONST', node.value) @@ -895,37 +979,44 @@ def visitGlobal(self, node): def visitName(self, node): self.set_lineno(node) - self.loadName(node.name) + if isinstance(node.ctx, ast.Store): + self.storeName(node.id) + elif isinstance(node.ctx, ast.Del): + self.delName(node.id) + else: + self.loadName(node.id) def visitPass(self, node): self.set_lineno(node) def visitImport(self, node): self.set_lineno(node) - level = 0 if self.graph.checkFlag(CO_FUTURE_ABSIMPORT) else -1 - for name, alias in node.names: + level = 0 + for alias in node.names: + name = alias.name + asname = alias.asname if VERSION > 1: self.emit('LOAD_CONST', level) self.emit('LOAD_CONST', None) self.emit('IMPORT_NAME', name) mod = name.split(".")[0] - if alias: + if asname: self._resolveDots(name) - self.storeName(alias) + self.storeName(asname) else: self.storeName(mod) - def visitFrom(self, node): + def visitImportFrom(self, node): self.set_lineno(node) level = node.level - if level == 0 and not self.graph.checkFlag(CO_FUTURE_ABSIMPORT): - level = -1 - fromlist = tuple(name for (name, alias) in node.names) + fromlist = tuple(alias.name for alias in node.names) if VERSION > 1: self.emit('LOAD_CONST', level) self.emit('LOAD_CONST', fromlist) - self.emit('IMPORT_NAME', node.modname) - for name, alias in node.names: + self.emit('IMPORT_NAME', node.module) + for alias in node.names: + name = alias.name + asname = alias.asname if VERSION > 1: if name == '*': self.namespace = 0 @@ -936,7 +1027,7 @@ def visitFrom(self, node): else: self.emit('IMPORT_FROM', name) self._resolveDots(name) - self.storeName(alias or name) + self.storeName(asname or name) else: self.emit('IMPORT_FROM', name) self.emit('POP_TOP') @@ -948,21 +1039,26 @@ def _resolveDots(self, name): for elt in elts[1:]: self.emit('LOAD_ATTR', elt) - def visitGetattr(self, node): - self.visit(node.expr) - self.emit('LOAD_ATTR', self.mangle(node.attrname)) + def visitAttribute(self, node): + self.visit(node.value) + if isinstance(node.ctx, ast.Store): + self.emit('STORE_ATTR', self.mangle(node.attr)) + elif isinstance(node.ctx, ast.Del): + self.emit('DELETE_ATTR', self.mangle(node.attr)) + else: + self.emit('LOAD_ATTR', self.mangle(node.attr)) # next five implement assignments def visitAssign(self, node): self.set_lineno(node) - self.visit(node.expr) - dups = len(node.nodes) - 1 - for i in range(len(node.nodes)): - elt = node.nodes[i] + self.visit(node.value) + dups = len(node.targets) - 1 + for i in range(len(node.targets)): + elt = node.targets[i] if i < dups: self.emit('DUP_TOP') - if isinstance(elt, ast.Node): + if isinstance(elt, ast.AST): self.visit(elt) def visitAssName(self, node): @@ -1006,32 +1102,33 @@ def visitAssList(self, node): def visitAugAssign(self, node): self.set_lineno(node) - aug_node = wrap_aug(node.node) + aug_node = wrap_aug(node.target) self.visit(aug_node, "load") - self.visit(node.expr) - self.emit(self._augmented_opcode[node.op]) + self.visit(node.value) + self.emit(self._augmented_opcode[type(node.op)]) self.visit(aug_node, "store") _augmented_opcode = { - '+=' : 'INPLACE_ADD', - '-=' : 'INPLACE_SUBTRACT', - '*=' : 'INPLACE_MULTIPLY', - '/=' : 'INPLACE_DIVIDE', - '//=': 'INPLACE_FLOOR_DIVIDE', - '%=' : 'INPLACE_MODULO', - '**=': 'INPLACE_POWER', - '>>=': 'INPLACE_RSHIFT', - '<<=': 'INPLACE_LSHIFT', - '&=' : 'INPLACE_AND', - '^=' : 'INPLACE_XOR', - '|=' : 'INPLACE_OR', + ast.Add: 'INPLACE_ADD', + ast.Sub: 'INPLACE_SUBTRACT', + ast.Mult: 'INPLACE_MULTIPLY', + ast.MatMult: 'INPLACE_MATRIX_MULTIPLY', + ast.Div: 'INPLACE_TRUE_DIVIDE', + ast.FloorDiv: 'INPLACE_FLOOR_DIVIDE', + ast.Mod: 'INPLACE_MODULO', + ast.Pow: 'INPLACE_POWER', + ast.RShift: 'INPLACE_RSHIFT', + ast.LShift: 'INPLACE_LSHIFT', + ast.BitAnd: 'INPLACE_AND', + ast.BitXor: 'INPLACE_XOR', + ast.BitOr: 'INPLACE_OR', } def visitAugName(self, node, mode): if mode == "load": - self.loadName(node.name) + self.loadName(node.id) elif mode == "store": - self.storeName(node.name) + self.storeName(node.id) def visitAugGetattr(self, node, mode): if mode == "load": @@ -1128,52 +1225,26 @@ def visitReturn(self, node): def visitYield(self, node): self.set_lineno(node) - self.visit(node.value) + if node.value: + self.visit(node.value) + else: + self.emit('LOAD_CONST', None) self.emit('YIELD_VALUE') # slice and subscript stuff - def visitSlice(self, node, aug_flag=None): - # aug_flag is used by visitAugSlice - self.visit(node.expr) - slice = 0 - if node.lower: - self.visit(node.lower) - slice = slice | 1 - if node.upper: - self.visit(node.upper) - slice = slice | 2 - if aug_flag: - if slice == 0: - self.emit('DUP_TOP') - elif slice == 3: - self.emit('DUP_TOPX', 3) - else: - self.emit('DUP_TOPX', 2) - if node.flags == 'OP_APPLY': - self.emit('SLICE+%d' % slice) - elif node.flags == 'OP_ASSIGN': - self.emit('STORE_SLICE+%d' % slice) - elif node.flags == 'OP_DELETE': - self.emit('DELETE_SLICE+%d' % slice) - else: - print("weird slice", node.flags) - raise - def visitSubscript(self, node, aug_flag=None): - self.visit(node.expr) - for sub in node.subs: - self.visit(sub) - if len(node.subs) > 1: - self.emit('BUILD_TUPLE', len(node.subs)) - if aug_flag: - self.emit('DUP_TOPX', 2) - if node.flags == 'OP_APPLY': + self.visit(node.value) + self.visit(node.slice) + + if isinstance(node.ctx, ast.Load): self.emit('BINARY_SUBSCR') - elif node.flags == 'OP_ASSIGN': + elif isinstance(node.ctx, ast.Store): self.emit('STORE_SUBSCR') - elif node.flags == 'OP_DELETE': + elif isinstance(node.ctx, ast.Del): self.emit('DELETE_SUBSCR') + else: + assert 0 # binary ops @@ -1182,74 +1253,47 @@ def binaryOp(self, node, op): self.visit(node.right) self.emit(op) - def visitAdd(self, node): - return self.binaryOp(node, 'BINARY_ADD') - - def visitSub(self, node): - return self.binaryOp(node, 'BINARY_SUBTRACT') - - def visitMul(self, node): - return self.binaryOp(node, 'BINARY_MULTIPLY') - - def visitDiv(self, node): - return self.binaryOp(node, self._div_op) - - def visitFloorDiv(self, node): - return self.binaryOp(node, 'BINARY_FLOOR_DIVIDE') - - def visitMod(self, node): - return self.binaryOp(node, 'BINARY_MODULO') - - def visitPower(self, node): - return self.binaryOp(node, 'BINARY_POWER') - - def visitLeftShift(self, node): - return self.binaryOp(node, 'BINARY_LSHIFT') + _binary_opcode = { + ast.Add: "BINARY_ADD", + ast.Sub: "BINARY_SUBTRACT", + ast.Mult: "BINARY_MULTIPLY", + ast.MatMult: "BINARY_MATRIX_MULTIPLY", + ast.Div: "BINARY_TRUE_DIVIDE", + ast.FloorDiv: "BINARY_FLOOR_DIVIDE", + ast.Mod: "BINARY_MODULO", + ast.Pow: "BINARY_POWER", + ast.LShift: "BINARY_LSHIFT", + ast.RShift: "BINARY_RSHIFT", + ast.BitOr: "BINARY_OR", + ast.BitXor: "BINARY_XOR", + ast.BitAnd: "BINARY_AND", + } - def visitRightShift(self, node): - return self.binaryOp(node, 'BINARY_RSHIFT') + def visitBinOp(self, node): + self.visit(node.left) + self.visit(node.right) + op = self._binary_opcode[type(node.op)] + self.emit(op) # unary ops def unaryOp(self, node, op): - self.visit(node.expr) + self.visit(node.operand) self.emit(op) - def visitInvert(self, node): - return self.unaryOp(node, 'UNARY_INVERT') - - def visitUnarySub(self, node): - return self.unaryOp(node, 'UNARY_NEGATIVE') - - def visitUnaryAdd(self, node): - return self.unaryOp(node, 'UNARY_POSITIVE') - - def visitUnaryInvert(self, node): - return self.unaryOp(node, 'UNARY_INVERT') + _unary_opcode = { + ast.Invert: "UNARY_INVERT", + ast.USub: "UNARY_NEGATIVE", + ast.UAdd: "UNARY_POSITIVE", + ast.Not: "UNARY_NOT", + } - def visitNot(self, node): - return self.unaryOp(node, 'UNARY_NOT') + def visitUnaryOp(self, node): + self.unaryOp(node, self._unary_opcode[type(node.op)]) def visitBackquote(self, node): return self.unaryOp(node, 'UNARY_CONVERT') - # bit ops - - def bitOp(self, nodes, op): - self.visit(nodes[0]) - for node in nodes[1:]: - self.visit(node) - self.emit(op) - - def visitBitand(self, node): - return self.bitOp(node.nodes, 'BINARY_AND') - - def visitBitor(self, node): - return self.bitOp(node.nodes, 'BINARY_OR') - - def visitBitxor(self, node): - return self.bitOp(node.nodes, 'BINARY_XOR') - # object constructors def visitEllipsis(self, node): @@ -1257,37 +1301,56 @@ def visitEllipsis(self, node): def visitTuple(self, node): self.set_lineno(node) - for elt in node.nodes: + for elt in node.elts: self.visit(elt) - self.emit('BUILD_TUPLE', len(node.nodes)) + self.emit('BUILD_TUPLE', len(node.elts)) def visitList(self, node): self.set_lineno(node) - for elt in node.nodes: + for elt in node.elts: self.visit(elt) - self.emit('BUILD_LIST', len(node.nodes)) + self.emit('BUILD_LIST', len(node.elts)) def visitSet(self, node): self.set_lineno(node) - for elt in node.nodes: + for elt in node.elts: self.visit(elt) - self.emit('BUILD_SET', len(node.nodes)) + self.emit('BUILD_SET', len(node.elts)) - def visitSliceobj(self, node): - for child in node.nodes: - self.visit(child) - self.emit('BUILD_SLICE', len(node.nodes)) - - def visitDict(self, node): + def visitSlice(self, node): + num = 2 + if node.lower: + self.visit(node.lower) + else: + self.emit('LOAD_CONST', None) + if node.upper: + self.visit(node.upper) + else: + self.emit('LOAD_CONST', None) + if node.step: + self.visit(node.step) + num += 1 + self.emit('BUILD_SLICE', num) + + # Create dict item by item. Saves interp stack size at the expense + # of bytecode size/speed. + def visitDict_by_one(self, node): self.set_lineno(node) self.emit('BUILD_MAP', 0) - for k, v in node.items: + for k, v in zip(node.keys, node.values): self.emit('DUP_TOP') self.visit(k) self.visit(v) self.emit('ROT_THREE') self.emit('STORE_SUBSCR') + def visitDict(self, node): + self.set_lineno(node) + for k, v in zip(node.keys, node.values): + self.visit(k) + self.visit(v) + self.emit('BUILD_MAP', len(node.keys)) + class NestedScopeMixin: """Defines initClass() for nested scoping (Python 2.2-compatible)""" def initClass(self): diff --git a/compiler/symbols.py b/compiler/symbols.py index 53448c2..c8bf310 100644 --- a/compiler/symbols.py +++ b/compiler/symbols.py @@ -1,7 +1,7 @@ """Module symbol-table generator""" from __future__ import print_function -from compiler import ast +import ast from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \ SC_FREE, SC_CELL, SC_UNKNOWN from compiler.misc import mangle @@ -223,35 +223,42 @@ def __init__(self): def visitModule(self, node): scope = self.module = self.scopes[node] = ModuleScope() - self.visit(node.node, scope) + self.visit(node.body, scope) visitExpression = visitModule - def visitFunction(self, node, parent): - if node.decorators: - self.visit(node.decorators, parent) + def visitFunctionDef(self, node, parent): + if node.decorator_list: + self.visit(node.decorator_list, parent) parent.add_def(node.name) - for n in node.defaults: + for n in node.args.defaults: self.visit(n, parent) scope = FunctionScope(node.name, self.module, self.klass) if parent.nested or isinstance(parent, FunctionScope): scope.nested = 1 self.scopes[node] = scope - self._do_args(scope, node.argnames) - self.visit(node.code, scope) + self._do_args(scope, node.args) + self.visit(node.body, scope) self.handle_free_vars(scope, parent) - def visitGenExpr(self, node, parent): + def visitGeneratorExp(self, node, parent): scope = GenExprScope(self.module, self.klass); if parent.nested or isinstance(parent, FunctionScope) \ or isinstance(parent, GenExprScope): scope.nested = 1 self.scopes[node] = scope - self.visit(node.code, scope) + for comp in node.generators: + self.visit(comp, scope) self.handle_free_vars(scope, parent) + def visitcomprehension(self, node, scope): + self.visit(node.target, scope, 1) + self.visit(node.iter, scope) + for if_ in node.ifs: + self.visit(if_, scope) + def visitGenExprInner(self, node, scope): for genfor in node.quals: self.visit(genfor, scope) @@ -273,41 +280,44 @@ def visitLambda(self, node, parent, assign=0): # any code that has a lambda on the left-hand side. assert not assign - for n in node.defaults: + for n in node.args.defaults: self.visit(n, parent) scope = LambdaScope(self.module, self.klass) if parent.nested or isinstance(parent, FunctionScope): scope.nested = 1 self.scopes[node] = scope - self._do_args(scope, node.argnames) - self.visit(node.code, scope) + self._do_args(scope, node.args) + self.visit(node.body, scope) self.handle_free_vars(scope, parent) def _do_args(self, scope, args): - for name in args: - if type(name) == types.TupleType: - self._do_args(scope, name) - else: - scope.add_param(name) + for arg in args.args: + name = arg.arg + scope.add_param(name) + if args.vararg: + scope.add_param(args.vararg.arg) + if args.kwarg: + scope.add_param(args.kwarg.arg) def handle_free_vars(self, scope, parent): parent.add_child(scope) scope.handle_children() - def visitClass(self, node, parent): + def visitClassDef(self, node, parent): parent.add_def(node.name) for n in node.bases: self.visit(n, parent) scope = ClassScope(node.name, self.module) if parent.nested or isinstance(parent, FunctionScope): scope.nested = 1 - if node.doc is not None: + doc = ast.get_docstring(node) + if doc is not None: scope.add_def('__doc__') scope.add_def('__module__') self.scopes[node] = scope prev = self.klass self.klass = node.name - self.visit(node.code, scope) + self.visit(node.body, scope) self.klass = prev self.handle_free_vars(scope, parent) @@ -319,31 +329,32 @@ def visitClass(self, node, parent): def visitName(self, node, scope, assign=0): if assign: - scope.add_def(node.name) + scope.add_def(node.id) else: - scope.add_use(node.name) + scope.add_use(node.id) # operations that bind new names def visitFor(self, node, scope): - self.visit(node.assign, scope, 1) - self.visit(node.list, scope) + self.visit(node.target, scope, 1) + self.visit(node.iter, scope) self.visit(node.body, scope) - if node.else_: - self.visit(node.else_, scope) + if node.orelse: + self.visit(node.orelse, scope) - def visitFrom(self, node, scope): - for name, asname in node.names: - if name == "*": + def visitImportFrom(self, node, scope): + for alias in node.names: + if alias.name == "*": continue - scope.add_def(asname or name) + scope.add_def(alias.asname or alias.name) def visitImport(self, node, scope): - for name, asname in node.names: + for alias in node.names: + name = alias.name i = name.find(".") if i > -1: name = name[:i] - scope.add_def(asname or name) + scope.add_def(alias.asname or name) def visitGlobal(self, node, scope): for name in node.names: @@ -362,9 +373,9 @@ def visitAssign(self, node, scope): visitor handles these nodes specially; they do not propagate the assign flag to their children. """ - for n in node.nodes: + for n in node.targets: self.visit(n, scope, 1) - self.visit(node.expr, scope) + self.visit(node.value, scope) def visitAssName(self, node, scope, assign=1): scope.add_def(node.name) @@ -373,45 +384,48 @@ def visitAssAttr(self, node, scope, assign=0): self.visit(node.expr, scope, 0) def visitSubscript(self, node, scope, assign=0): - self.visit(node.expr, scope, 0) - for n in node.subs: - self.visit(n, scope, 0) + self.visit(node.value, scope, 0) + self.visit(node.slice, scope, 0) def visitSlice(self, node, scope, assign=0): - self.visit(node.expr, scope, 0) if node.lower: self.visit(node.lower, scope, 0) if node.upper: self.visit(node.upper, scope, 0) + if node.step: + self.visit(node.step, scope, 0) def visitAugAssign(self, node, scope): # If the LHS is a name, then this counts as assignment. # Otherwise, it's just use. - self.visit(node.node, scope) - if isinstance(node.node, ast.Name): - self.visit(node.node, scope, 1) # XXX worry about this - self.visit(node.expr, scope) + self.visit(node.target, scope) + if isinstance(node.target, ast.Name): + self.visit(node.target, scope, 1) # XXX worry about this + self.visit(node.value, scope) # prune if statements if tests are false _const_types = str, bytes, int, long, float def visitIf(self, node, scope): - for test, body in node.tests: - if isinstance(test, ast.Const): - if type(test.value) in self._const_types: - if not test.value: - continue - self.visit(test, scope) - self.visit(body, scope) - if node.else_: - self.visit(node.else_, scope) + self.visit(node.test, scope) + self.visit(node.body, scope) + if node.orelse: + self.visit(node.orelse, scope) # a yield statement signals a generator def visitYield(self, node, scope): scope.generator = 1 - self.visit(node.value, scope) + if node.value: + self.visit(node.value, scope) + + def visitTry(self, node, scope): + # Handle exception capturing vars + for handler in node.handlers: + if handler.name: + scope.add_def(handler.name) + def list_eq(l1, l2): return sorted(l1) == sorted(l2) diff --git a/dis27.py b/dis27.py new file mode 100644 index 0000000..6e112e2 --- /dev/null +++ b/dis27.py @@ -0,0 +1,226 @@ +"""Disassembler of Python byte code into mnemonics.""" + +import sys +import types + +from opcode import * +from opcode import __all__ as _opcodes_all + +__all__ = ["dis", "disassemble", "distb", "disco", + "findlinestarts", "findlabels"] + _opcodes_all +del _opcodes_all + +_have_code = (types.MethodType, types.FunctionType, types.CodeType, + types.ClassType, type) + +def dis(x=None): + """Disassemble classes, methods, functions, or code. + + With no argument, disassemble the last traceback. + + """ + if x is None: + distb() + return + if isinstance(x, types.InstanceType): + x = x.__class__ + if hasattr(x, 'im_func'): + x = x.im_func + if hasattr(x, 'func_code'): + x = x.func_code + if hasattr(x, '__dict__'): + items = x.__dict__.items() + items.sort() + for name, x1 in items: + if isinstance(x1, _have_code): + print "Disassembly of %s:" % name + try: + dis(x1) + except TypeError, msg: + print "Sorry:", msg + print + elif hasattr(x, 'co_code'): + disassemble(x) + elif isinstance(x, str): + disassemble_string(x) + else: + raise TypeError, \ + "don't know how to disassemble %s objects" % \ + type(x).__name__ + +def distb(tb=None): + """Disassemble a traceback (default: last traceback).""" + if tb is None: + try: + tb = sys.last_traceback + except AttributeError: + raise RuntimeError, "no last traceback to disassemble" + while tb.tb_next: tb = tb.tb_next + disassemble(tb.tb_frame.f_code, tb.tb_lasti) + +const_repr = repr + +def disassemble(co, lasti=-1): + """Disassemble a code object.""" + code = co.co_code + labels = findlabels(code) + linestarts = dict(findlinestarts(co)) + n = len(code) + i = 0 + extended_arg = 0 + free = None + while i < n: + c = code[i] + op = ord(c) + if i in linestarts: + if i > 0: + print + print "%3d" % linestarts[i], + else: + print ' ', + + if i == lasti: print '-->', + else: print ' ', + if i in labels: print '>>', + else: print ' ', + print repr(i).rjust(4), + print opname[op].ljust(20), + i = i+1 + if op >= HAVE_ARGUMENT: + oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg + extended_arg = 0 + i = i+2 + if op == EXTENDED_ARG: + extended_arg = oparg*65536L + print repr(oparg).rjust(5), + if op in hasconst: + print '(' + const_repr(co.co_consts[oparg]) + ')', + elif op in hasname: + print '(' + co.co_names[oparg] + ')', + elif op in hasjrel: + print '(to ' + repr(i + oparg) + ')', + elif op in haslocal: + print '(' + co.co_varnames[oparg] + ')', + elif op in hascompare: + print '(' + cmp_op[oparg] + ')', + elif op in hasfree: + if free is None: + free = co.co_cellvars + co.co_freevars + print '(' + free[oparg] + ')', + print + +def disassemble_string(code, lasti=-1, varnames=None, names=None, + constants=None): + labels = findlabels(code) + n = len(code) + i = 0 + while i < n: + c = code[i] + op = ord(c) + if i == lasti: print '-->', + else: print ' ', + if i in labels: print '>>', + else: print ' ', + print repr(i).rjust(4), + print opname[op].ljust(15), + i = i+1 + if op >= HAVE_ARGUMENT: + oparg = ord(code[i]) + ord(code[i+1])*256 + i = i+2 + print repr(oparg).rjust(5), + if op in hasconst: + if constants: + print '(' + repr(constants[oparg]) + ')', + else: + print '(%d)'%oparg, + elif op in hasname: + if names is not None: + print '(' + names[oparg] + ')', + else: + print '(%d)'%oparg, + elif op in hasjrel: + print '(to ' + repr(i + oparg) + ')', + elif op in haslocal: + if varnames: + print '(' + varnames[oparg] + ')', + else: + print '(%d)' % oparg, + elif op in hascompare: + print '(' + cmp_op[oparg] + ')', + print + +disco = disassemble # XXX For backwards compatibility + +def findlabels(code): + """Detect all offsets in a byte code which are jump targets. + + Return the list of offsets. + + """ + labels = [] + n = len(code) + i = 0 + while i < n: + c = code[i] + op = ord(c) + i = i+1 + if op >= HAVE_ARGUMENT: + oparg = ord(code[i]) + ord(code[i+1])*256 + i = i+2 + label = -1 + if op in hasjrel: + label = i+oparg + elif op in hasjabs: + label = oparg + if label >= 0: + if label not in labels: + labels.append(label) + return labels + +def findlinestarts(code): + """Find the offsets in a byte code which are start of lines in the source. + + Generate pairs (offset, lineno) as described in Python/compile.c. + + """ + byte_increments = [ord(c) for c in code.co_lnotab[0::2]] + line_increments = [ord(c) for c in code.co_lnotab[1::2]] + + lastlineno = None + lineno = code.co_firstlineno + addr = 0 + for byte_incr, line_incr in zip(byte_increments, line_increments): + if byte_incr: + if lineno != lastlineno: + yield (addr, lineno) + lastlineno = lineno + addr += byte_incr + lineno += line_incr + if lineno != lastlineno: + yield (addr, lineno) + +def _test(): + """Simple test program to disassemble a file.""" + if sys.argv[1:]: + if sys.argv[2:]: + sys.stderr.write("usage: python dis.py [-|file]\n") + sys.exit(2) + fn = sys.argv[1] + if not fn or fn == "-": + fn = None + else: + fn = None + if fn is None: + f = sys.stdin + else: + f = open(fn) + source = f.read() + if fn is not None: + f.close() + else: + fn = "" + code = compile(source, fn, "exec") + dis(code) + +if __name__ == "__main__": + _test()