diff options
Diffstat (limited to 'Lib/compiler')
| -rw-r--r-- | Lib/compiler/__init__.py | 8 | ||||
| -rw-r--r-- | Lib/compiler/ast.py | 61 | ||||
| -rw-r--r-- | Lib/compiler/consts.py | 9 | ||||
| -rw-r--r-- | Lib/compiler/pyassem.py | 263 | ||||
| -rw-r--r-- | Lib/compiler/pycodegen.py | 123 | ||||
| -rw-r--r-- | Lib/compiler/symbols.py | 9 | ||||
| -rw-r--r-- | Lib/compiler/transformer.py | 274 |
7 files changed, 409 insertions, 338 deletions
diff --git a/Lib/compiler/__init__.py b/Lib/compiler/__init__.py index d75140aa72..2a6f64fa50 100644 --- a/Lib/compiler/__init__.py +++ b/Lib/compiler/__init__.py @@ -20,9 +20,11 @@ compile(source, filename, mode, flags=None, dont_inherit=None) compileFile(filename) Generates a .pyc file by compiling filename. """ -from warnings import warnpy3k -warnpy3k("the compiler package has been removed in Python 3.0", stacklevel=2) -del warnpy3k + +import warnings + +warnings.warn("The compiler package is deprecated and removed in Python 3.x.", + DeprecationWarning, stacklevel=2) from compiler.transformer import parse, parseFile from compiler.visitor import walk diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py index f92307759e..4c3fc161d3 100644 --- a/Lib/compiler/ast.py +++ b/Lib/compiler/ast.py @@ -890,6 +890,51 @@ class ListCompIf(Node): def __repr__(self): return "ListCompIf(%s)" % (repr(self.test),) +class SetComp(Node): + def __init__(self, expr, quals, lineno=None): + self.expr = expr + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "SetComp(%s, %s)" % (repr(self.expr), repr(self.quals)) + +class DictComp(Node): + def __init__(self, key, value, quals, lineno=None): + self.key = key + self.value = value + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.key) + children.append(self.value) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.key) + nodelist.append(self.value) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "DictComp(%s, %s, %s)" % (repr(self.key), repr(self.value), repr(self.quals)) + class Mod(Node): def __init__(self, leftright, lineno=None): self.left = leftright[0] @@ -1107,6 +1152,22 @@ class RightShift(Node): def __repr__(self): return "RightShift((%s, %s))" % (repr(self.left), repr(self.right)) +class Set(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Set(%s)" % (repr(self.nodes),) + class Slice(Node): def __init__(self, expr, flags, lower, upper, lineno=None): self.expr = expr diff --git a/Lib/compiler/consts.py b/Lib/compiler/consts.py index dd42793aa9..022f6daa4c 100644 --- a/Lib/compiler/consts.py +++ b/Lib/compiler/consts.py @@ -4,10 +4,11 @@ OP_DELETE = 'OP_DELETE' OP_APPLY = 'OP_APPLY' SC_LOCAL = 1 -SC_GLOBAL = 2 -SC_FREE = 3 -SC_CELL = 4 -SC_UNKNOWN = 5 +SC_GLOBAL_IMPLICIT = 2 +SC_GLOBAL_EXPLICT = 3 +SC_FREE = 4 +SC_CELL = 5 +SC_UNKNOWN = 6 CO_OPTIMIZED = 0x0001 CO_NEWLOCALS = 0x0002 diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py index 6efec28094..286be0c8c7 100644 --- a/Lib/compiler/pyassem.py +++ b/Lib/compiler/pyassem.py @@ -21,6 +21,7 @@ class FlowGraph: if self.current: print "end", repr(self.current) print " next", self.current.next + print " prev", self.current.prev print " ", self.current.get_children() print repr(block) self.current = block @@ -40,13 +41,12 @@ class FlowGraph: if block is None: block = self.newBlock() - # Note: If the current block ends with an unconditional - # control transfer, then it is incorrect to add an implicit - # transfer to the block graph. The current code requires - # these edges to get the blocks emitted in the right order, - # however. :-( If a client needs to remove these edges, call - # pruneEdges(). - + # Note: If the current block ends with an unconditional control + # transfer, then it is techically incorrect to add an implicit + # transfer to the block graph. Doing so results in code generation + # for unreachable blocks. That doesn't appear to be very common + # with Python code and since the built-in compiler doesn't optimize + # it out we don't either. self.current.addNext(block) self.startBlock(block) @@ -69,8 +69,6 @@ class FlowGraph: def emit(self, *inst): if self._debug: print "\t", inst - if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']: - self.current.addOutEdge(self.exit) if len(inst) == 2 and isinstance(inst[1], Block): self.current.addOutEdge(inst[1]) self.current.emit(inst) @@ -80,118 +78,9 @@ class FlowGraph: i.e. each node appears before all of its successors """ - # XXX make sure every node that doesn't have an explicit next - # is set so that next points to exit - for b in self.blocks.elements(): - if b is self.exit: - continue - if not b.next: - b.addNext(self.exit) - order = dfs_postorder(self.entry, {}) - order.reverse() - self.fixupOrder(order, self.exit) - # hack alert - if not self.exit in order: - order.append(self.exit) - + order = order_blocks(self.entry, self.exit) return order - def fixupOrder(self, blocks, default_next): - """Fixup bad order introduced by DFS.""" - - # XXX This is a total mess. There must be a better way to get - # the code blocks in the right order. - - self.fixupOrderHonorNext(blocks, default_next) - self.fixupOrderForward(blocks, default_next) - - def fixupOrderHonorNext(self, blocks, default_next): - """Fix one problem with DFS. - - The DFS uses child block, but doesn't know about the special - "next" block. As a result, the DFS can order blocks so that a - block isn't next to the right block for implicit control - transfers. - """ - index = {} - for i in range(len(blocks)): - index[blocks[i]] = i - - for i in range(0, len(blocks) - 1): - b = blocks[i] - n = blocks[i + 1] - if not b.next or b.next[0] == default_next or b.next[0] == n: - continue - # The blocks are in the wrong order. Find the chain of - # blocks to insert where they belong. - cur = b - chain = [] - elt = cur - while elt.next and elt.next[0] != default_next: - chain.append(elt.next[0]) - elt = elt.next[0] - # Now remove the blocks in the chain from the current - # block list, so that they can be re-inserted. - l = [] - for b in chain: - assert index[b] > i - l.append((index[b], b)) - l.sort() - l.reverse() - for j, b in l: - del blocks[index[b]] - # Insert the chain in the proper location - blocks[i:i + 1] = [cur] + chain - # Finally, re-compute the block indexes - for i in range(len(blocks)): - index[blocks[i]] = i - - def fixupOrderForward(self, blocks, default_next): - """Make sure all JUMP_FORWARDs jump forward""" - index = {} - chains = [] - cur = [] - for b in blocks: - index[b] = len(chains) - cur.append(b) - if b.next and b.next[0] == default_next: - chains.append(cur) - cur = [] - chains.append(cur) - - while 1: - constraints = [] - - for i in range(len(chains)): - l = chains[i] - for b in l: - for c in b.get_children(): - if index[c] < i: - forward_p = 0 - for inst in b.insts: - if inst[0] == 'JUMP_FORWARD': - if inst[1] == c: - forward_p = 1 - if not forward_p: - continue - constraints.append((index[c], i)) - - if not constraints: - break - - # XXX just do one for now - # do swaps to get things in the right order - goes_before, a_chain = constraints[0] - assert a_chain > goes_before - c = chains[a_chain] - chains.remove(c) - chains.insert(goes_before, c) - - del blocks[:] - for c in chains: - for b in c: - blocks.append(b) - def getBlocks(self): return self.blocks.elements() @@ -205,27 +94,84 @@ class FlowGraph: l.extend(b.getContainedGraphs()) return l -def dfs_postorder(b, seen): - """Depth-first search of tree rooted at b, return in postorder""" + +def order_blocks(start_block, exit_block): + """Order blocks so that they are emitted in the right order""" + # Rules: + # - when a block has a next block, the next block must be emitted just after + # - when a block has followers (relative jumps), it must be emitted before + # them + # - all reachable blocks must be emitted order = [] - seen[b] = b - for c in b.get_children(): - if c in seen: + + # Find all the blocks to be emitted. + remaining = set() + todo = [start_block] + while todo: + b = todo.pop() + if b in remaining: continue - order = order + dfs_postorder(c, seen) - order.append(b) + remaining.add(b) + for c in b.get_children(): + if c not in remaining: + todo.append(c) + + # A block is dominated by another block if that block must be emitted + # before it. + dominators = {} + for b in remaining: + if __debug__ and b.next: + assert b is b.next[0].prev[0], (b, b.next) + # Make sure every block appears in dominators, even if no + # other block must precede it. + dominators.setdefault(b, set()) + # preceeding blocks dominate following blocks + for c in b.get_followers(): + while 1: + dominators.setdefault(c, set()).add(b) + # Any block that has a next pointer leading to c is also + # dominated because the whole chain will be emitted at once. + # Walk backwards and add them all. + if c.prev and c.prev[0] is not b: + c = c.prev[0] + else: + break + + def find_next(): + # Find a block that can be emitted next. + for b in remaining: + for c in dominators[b]: + if c in remaining: + break # can't emit yet, dominated by a remaining block + else: + return b + assert 0, 'circular dependency, cannot find next block' + + b = start_block + while 1: + order.append(b) + remaining.discard(b) + if b.next: + b = b.next[0] + continue + elif b is not exit_block and not b.has_unconditional_transfer(): + order.append(exit_block) + if not remaining: + break + b = find_next() return order + class Block: _count = 0 def __init__(self, label=''): self.insts = [] - self.inEdges = misc.Set() - self.outEdges = misc.Set() + self.outEdges = set() self.label = label self.bid = Block._count self.next = [] + self.prev = [] Block._count = Block._count + 1 def __repr__(self): @@ -241,51 +187,46 @@ class Block: def emit(self, inst): op = inst[0] - if op[:4] == 'JUMP': - self.outEdges.add(inst[1]) self.insts.append(inst) def getInstructions(self): return self.insts - def addInEdge(self, block): - self.inEdges.add(block) - def addOutEdge(self, block): self.outEdges.add(block) def addNext(self, block): self.next.append(block) assert len(self.next) == 1, map(str, self.next) + block.prev.append(self) + assert len(block.prev) == 1, map(str, block.prev) - _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE', - 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP') - - def pruneNext(self): - """Remove bogus edge for unconditional transfers + _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', + 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP', + ) - Each block has a next edge that accounts for implicit control - transfers, e.g. from a JUMP_IF_FALSE to the block that will be - executed if the test is true. - - These edges must remain for the current assembler code to - work. If they are removed, the dfs_postorder gets things in - weird orders. However, they shouldn't be there for other - purposes, e.g. conversion to SSA form. This method will - remove the next edge when it follows an unconditional control - transfer. - """ + def has_unconditional_transfer(self): + """Returns True if there is an unconditional transfer to an other block + at the end of this block. This means there is no risk for the bytecode + executer to go past this block's bytecode.""" try: op, arg = self.insts[-1] except (IndexError, ValueError): return - if op in self._uncond_transfer: - self.next = [] + return op in self._uncond_transfer def get_children(self): - if self.next and self.next[0] in self.outEdges: - self.outEdges.remove(self.next[0]) - return self.outEdges.elements() + self.next + return list(self.outEdges) + self.next + + def get_followers(self): + """Get the whole list of followers, including the next block.""" + followers = set(self.next) + # Blocks that must be emitted *after* this one, because of + # bytecode offsets (e.g. relative jumps) pointing to them. + for inst in self.insts: + if inst[0] in PyFlowGraph.hasjrel: + followers.add(inst[1]) + return followers def getContainedGraphs(self): """Return all graphs contained within this block. @@ -446,18 +387,18 @@ class PyFlowGraph(FlowGraph): elif inst[0] != "SET_LINENO": pc = pc + 3 opname = inst[0] - if self.hasjrel.has_elt(opname): + if opname in self.hasjrel: oparg = inst[1] offset = begin[oparg] - pc insts[i] = opname, offset - elif self.hasjabs.has_elt(opname): + elif opname in self.hasjabs: insts[i] = opname, begin[inst[1]] self.stage = FLAT - hasjrel = misc.Set() + hasjrel = set() for i in dis.hasjrel: hasjrel.add(dis.opname[i]) - hasjabs = misc.Set() + hasjabs = set() for i in dis.hasjabs: hasjabs.add(dis.opname[i]) @@ -744,7 +685,9 @@ class StackDepthTracker: effect = { 'POP_TOP': -1, 'DUP_TOP': 1, - 'LIST_APPEND': -2, + 'LIST_APPEND': -1, + 'SET_ADD': -1, + 'MAP_ADD': -2, 'SLICE+1': -1, 'SLICE+2': -1, 'SLICE+3': -2, @@ -793,6 +736,8 @@ class StackDepthTracker: return -count+1 def BUILD_LIST(self, count): return -count+1 + def BUILD_SET(self, count): + return -count+1 def CALL_FUNCTION(self, argc): hi, lo = divmod(argc, 256) return -(lo + hi * 2) diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py index 17a8f4a388..4f2ecf2ae8 100644 --- a/Lib/compiler/pycodegen.py +++ b/Lib/compiler/pycodegen.py @@ -7,7 +7,8 @@ from cStringIO import StringIO from compiler import ast, parse, walk, syntax from compiler import pyassem, misc, future, symbols -from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL +from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \ + SC_FREE, SC_CELL from compiler.consts import (CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, CO_NESTED, CO_GENERATOR, CO_FUTURE_DIVISION, CO_FUTURE_ABSIMPORT, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_PRINT_FUNCTION) @@ -282,7 +283,9 @@ class CodeGenerator: self.emit(prefix + '_NAME', name) else: self.emit(prefix + '_FAST', name) - elif scope == SC_GLOBAL: + elif scope == SC_GLOBAL_EXPLICT: + self.emit(prefix + '_GLOBAL', name) + elif scope == SC_GLOBAL_IMPLICIT: if not self.optimized: self.emit(prefix + '_NAME', name) else: @@ -418,13 +421,11 @@ class CodeGenerator: self.set_lineno(test) self.visit(test) nextTest = self.newBlock() - self.emit('JUMP_IF_FALSE', nextTest) + self.emit('POP_JUMP_IF_FALSE', nextTest) self.nextBlock() - self.emit('POP_TOP') self.visit(suite) self.emit('JUMP_FORWARD', end) self.startBlock(nextTest) - self.emit('POP_TOP') if node.else_: self.visit(node.else_) self.nextBlock(end) @@ -443,15 +444,13 @@ class CodeGenerator: self.set_lineno(node, force=True) self.visit(node.test) - self.emit('JUMP_IF_FALSE', else_ or after) + self.emit('POP_JUMP_IF_FALSE', else_ or after) self.nextBlock() - self.emit('POP_TOP') self.visit(node.body) self.emit('JUMP_ABSOLUTE', loop) self.startBlock(else_) # or just the POPs if not else clause - self.emit('POP_TOP') self.emit('POP_BLOCK') self.setups.pop() if node.else_: @@ -522,26 +521,23 @@ class CodeGenerator: self.visit(child) self.emit(jump, end) self.nextBlock() - self.emit('POP_TOP') self.visit(node.nodes[-1]) self.nextBlock(end) def visitAnd(self, node): - self.visitTest(node, 'JUMP_IF_FALSE') + self.visitTest(node, 'JUMP_IF_FALSE_OR_POP') def visitOr(self, node): - self.visitTest(node, 'JUMP_IF_TRUE') + self.visitTest(node, 'JUMP_IF_TRUE_OR_POP') def visitIfExp(self, node): endblock = self.newBlock() elseblock = self.newBlock() self.visit(node.test) - self.emit('JUMP_IF_FALSE', elseblock) - self.emit('POP_TOP') + self.emit('POP_JUMP_IF_FALSE', elseblock) self.visit(node.then) self.emit('JUMP_FORWARD', endblock) self.nextBlock(elseblock) - self.emit('POP_TOP') self.visit(node.else_) self.nextBlock(endblock) @@ -553,9 +549,8 @@ class CodeGenerator: self.emit('DUP_TOP') self.emit('ROT_THREE') self.emit('COMPARE_OP', op) - self.emit('JUMP_IF_FALSE', cleanup) + self.emit('JUMP_IF_FALSE_OR_POP', cleanup) self.nextBlock() - self.emit('POP_TOP') # now do the last comparison if node.ops: op, code = node.ops[-1] @@ -570,16 +565,10 @@ class CodeGenerator: self.nextBlock(end) # list comprehensions - __list_count = 0 - def visitListComp(self, node): self.set_lineno(node) # setup list - tmpname = "$list%d" % self.__list_count - self.__list_count = self.__list_count + 1 self.emit('BUILD_LIST', 0) - self.emit('DUP_TOP') - self._implicitNameOp('STORE', tmpname) stack = [] for i, for_ in zip(range(len(node.quals)), node.quals): @@ -591,22 +580,63 @@ class CodeGenerator: self.visit(if_, cont) stack.insert(0, (start, cont, anchor)) - self._implicitNameOp('LOAD', tmpname) self.visit(node.expr) - self.emit('LIST_APPEND') + self.emit('LIST_APPEND', len(node.quals) + 1) for start, cont, anchor in stack: if cont: - skip_one = self.newBlock() - self.emit('JUMP_FORWARD', skip_one) - self.startBlock(cont) - self.emit('POP_TOP') - self.nextBlock(skip_one) + self.nextBlock(cont) self.emit('JUMP_ABSOLUTE', start) self.startBlock(anchor) - self._implicitNameOp('DELETE', tmpname) - self.__list_count = self.__list_count - 1 + def visitSetComp(self, node): + self.set_lineno(node) + # setup list + self.emit('BUILD_SET', 0) + + stack = [] + for i, for_ in zip(range(len(node.quals)), node.quals): + start, anchor = self.visit(for_) + cont = None + for if_ in for_.ifs: + if cont is None: + cont = self.newBlock() + self.visit(if_, cont) + stack.insert(0, (start, cont, anchor)) + + self.visit(node.expr) + self.emit('SET_ADD', len(node.quals) + 1) + + for start, cont, anchor in stack: + if cont: + self.nextBlock(cont) + self.emit('JUMP_ABSOLUTE', start) + self.startBlock(anchor) + + def visitDictComp(self, node): + self.set_lineno(node) + # setup list + self.emit('BUILD_MAP', 0) + + stack = [] + for i, for_ in zip(range(len(node.quals)), node.quals): + start, anchor = self.visit(for_) + cont = None + for if_ in for_.ifs: + if cont is None: + cont = self.newBlock() + self.visit(if_, cont) + stack.insert(0, (start, cont, anchor)) + + self.visit(node.value) + self.visit(node.key) + self.emit('MAP_ADD', len(node.quals) + 1) + + for start, cont, anchor in stack: + if cont: + self.nextBlock(cont) + self.emit('JUMP_ABSOLUTE', start) + self.startBlock(anchor) def visitListCompFor(self, node): start = self.newBlock() @@ -624,9 +654,8 @@ class CodeGenerator: def visitListCompIf(self, node, branch): self.set_lineno(node, force=True) self.visit(node.test) - self.emit('JUMP_IF_FALSE', branch) + self.emit('POP_JUMP_IF_FALSE', branch) self.newBlock() - self.emit('POP_TOP') def _makeClosure(self, gen, args): frees = gen.scope.get_free_vars() @@ -672,16 +701,12 @@ class CodeGenerator: for start, cont, anchor, end in stack: if cont: - skip_one = self.newBlock() - self.emit('JUMP_FORWARD', skip_one) - self.startBlock(cont) - self.emit('POP_TOP') - self.nextBlock(skip_one) + self.nextBlock(cont) self.emit('JUMP_ABSOLUTE', start) self.startBlock(anchor) self.emit('POP_BLOCK') self.setups.pop() - self.startBlock(end) + self.nextBlock(end) self.emit('LOAD_CONST', None) @@ -709,9 +734,8 @@ class CodeGenerator: def visitGenExprIf(self, node, branch): self.set_lineno(node, force=True) self.visit(node.test) - self.emit('JUMP_IF_FALSE', branch) + self.emit('POP_JUMP_IF_FALSE', branch) self.newBlock() - self.emit('POP_TOP') # exception related @@ -726,9 +750,8 @@ class CodeGenerator: # is a sort of renaming op. self.nextBlock() self.visit(node.test) - self.emit('JUMP_IF_TRUE', end) + self.emit('POP_JUMP_IF_TRUE', end) self.nextBlock() - self.emit('POP_TOP') self.emit('LOAD_GLOBAL', 'AssertionError') if node.fail: self.visit(node.fail) @@ -736,7 +759,6 @@ class CodeGenerator: else: self.emit('RAISE_VARARGS', 1) self.nextBlock(end) - self.emit('POP_TOP') def visitRaise(self, node): self.set_lineno(node) @@ -779,9 +801,8 @@ class CodeGenerator: self.visit(expr) self.emit('COMPARE_OP', 'exception match') next = self.newBlock() - self.emit('JUMP_IF_FALSE', next) + self.emit('POP_JUMP_IF_FALSE', next) self.nextBlock() - self.emit('POP_TOP') self.emit('POP_TOP') if target: self.visit(target) @@ -794,8 +815,6 @@ class CodeGenerator: self.nextBlock(next) else: self.nextBlock() - if expr: # XXX - self.emit('POP_TOP') self.emit('END_FINALLY') if node.else_: self.nextBlock(lElse) @@ -824,8 +843,8 @@ class CodeGenerator: def visitWith(self, node): body = self.newBlock() final = self.newBlock() - valuevar = "$value%d" % self.__with_count self.__with_count += 1 + valuevar = "_[%d]" % self.__with_count self.set_lineno(node) self.visit(node.expr) self.emit('DUP_TOP') @@ -1245,6 +1264,12 @@ class CodeGenerator: self.visit(elt) self.emit('BUILD_LIST', len(node.nodes)) + def visitSet(self, node): + self.set_lineno(node) + for elt in node.nodes: + self.visit(elt) + self.emit('BUILD_SET', len(node.nodes)) + def visitSliceobj(self, node): for child in node.nodes: self.visit(child) diff --git a/Lib/compiler/symbols.py b/Lib/compiler/symbols.py index 0e660ae7da..0bbdc7122d 100644 --- a/Lib/compiler/symbols.py +++ b/Lib/compiler/symbols.py @@ -1,7 +1,8 @@ """Module symbol-table generator""" from compiler import ast -from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN +from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \ + SC_FREE, SC_CELL, SC_UNKNOWN from compiler.misc import mangle import types @@ -89,7 +90,7 @@ class Scope: The scope of a name could be LOCAL, GLOBAL, FREE, or CELL. """ if name in self.globals: - return SC_GLOBAL + return SC_GLOBAL_EXPLICT if name in self.cells: return SC_CELL if name in self.defs: @@ -99,7 +100,7 @@ class Scope: if self.nested: return SC_UNKNOWN else: - return SC_GLOBAL + return SC_GLOBAL_IMPLICIT def get_free_vars(self): if not self.nested: @@ -152,7 +153,7 @@ class Scope: if sc == SC_UNKNOWN or sc == SC_FREE \ or isinstance(self, ClassScope): self.frees[name] = 1 - elif sc == SC_GLOBAL: + elif sc == SC_GLOBAL_IMPLICIT: child_globals.append(name) elif isinstance(self, FunctionScope) and sc == SC_LOCAL: self.cells[name] = 1 diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py index f5fe582bc9..d4f4613f48 100644 --- a/Lib/compiler/transformer.py +++ b/Lib/compiler/transformer.py @@ -581,8 +581,10 @@ class Transformer: testlist1 = testlist exprlist = testlist - def testlist_gexp(self, nodelist): - if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for: + def testlist_comp(self, nodelist): + # test ( comp_for | (',' test)* [','] ) + assert nodelist[0][0] == symbol.test + if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for: test = self.com_node(nodelist[0]) return self.com_generator_expression(test, nodelist[1]) return self.testlist(nodelist) @@ -749,7 +751,7 @@ class Transformer: def atom_lbrace(self, nodelist): if nodelist[1][0] == token.RBRACE: return Dict((), lineno=nodelist[0][2]) - return self.com_dictmaker(nodelist[1]) + return self.com_dictorsetmaker(nodelist[1]) def atom_backquote(self, nodelist): return Backquote(self.com_node(nodelist[1])) @@ -965,18 +967,22 @@ class Transformer: return try_except def com_with(self, nodelist): - # with_stmt: 'with' expr [with_var] ':' suite - expr = self.com_node(nodelist[1]) + # with_stmt: 'with' with_item (',' with_item)* ':' suite body = self.com_node(nodelist[-1]) - if nodelist[2][0] == token.COLON: - var = None + for i in range(len(nodelist) - 3, 0, -2): + ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) + if i == 1: + return ret + body = ret + + def com_with_item(self, nodelist, body, lineno): + # with_item: test ['as' expr] + if len(nodelist) == 4: + var = self.com_assign(nodelist[3], OP_ASSIGN) else: - var = self.com_assign(nodelist[2][2], OP_ASSIGN) - return With(expr, var, body, lineno=nodelist[0][2]) - - def com_with_var(self, nodelist): - # with_var: 'as' expr - return self.com_node(nodelist[1]) + var = None + expr = self.com_node(nodelist[1]) + return With(expr, var, body, lineno=lineno) def com_augassign_op(self, node): assert node[0] == symbol.augassign @@ -997,7 +1003,7 @@ class Transformer: # loop to avoid trivial recursion while 1: t = node[0] - if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp): + if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp): if len(node) > 2: return self.com_assign_tuple(node, assigning) node = node[1] @@ -1095,111 +1101,141 @@ class Transformer: else: stmts.append(result) - if hasattr(symbol, 'list_for'): - def com_list_constructor(self, nodelist): - # listmaker: test ( list_for | (',' test)* [','] ) - values = [] - for i in range(1, len(nodelist)): - if nodelist[i][0] == symbol.list_for: - assert len(nodelist[i:]) == 1 - return self.com_list_comprehension(values[0], - nodelist[i]) - elif nodelist[i][0] == token.COMMA: - continue - values.append(self.com_node(nodelist[i])) - return List(values, lineno=values[0].lineno) - - def com_list_comprehension(self, expr, node): - # list_iter: list_for | list_if - # list_for: 'for' exprlist 'in' testlist [list_iter] - # list_if: 'if' test [list_iter] - - # XXX should raise SyntaxError for assignment - - lineno = node[1][2] - fors = [] - while node: - t = node[1][1] - if t == 'for': - assignNode = self.com_assign(node[2], OP_ASSIGN) - listNode = self.com_node(node[4]) - newfor = ListCompFor(assignNode, listNode, []) - newfor.lineno = node[1][2] - fors.append(newfor) - if len(node) == 5: - node = None - else: - node = self.com_list_iter(node[5]) - elif t == 'if': - test = self.com_node(node[2]) - newif = ListCompIf(test, lineno=node[1][2]) - newfor.ifs.append(newif) - if len(node) == 3: - node = None - else: - node = self.com_list_iter(node[3]) + def com_list_constructor(self, nodelist): + # listmaker: test ( list_for | (',' test)* [','] ) + values = [] + for i in range(1, len(nodelist)): + if nodelist[i][0] == symbol.list_for: + assert len(nodelist[i:]) == 1 + return self.com_list_comprehension(values[0], + nodelist[i]) + elif nodelist[i][0] == token.COMMA: + continue + values.append(self.com_node(nodelist[i])) + return List(values, lineno=values[0].lineno) + + def com_list_comprehension(self, expr, node): + return self.com_comprehension(expr, None, node, 'list') + + def com_comprehension(self, expr1, expr2, node, type): + # list_iter: list_for | list_if + # list_for: 'for' exprlist 'in' testlist [list_iter] + # list_if: 'if' test [list_iter] + + # XXX should raise SyntaxError for assignment + # XXX(avassalotti) Set and dict comprehensions should have generator + # semantics. In other words, they shouldn't leak + # variables outside of the comprehension's scope. + + lineno = node[1][2] + fors = [] + while node: + t = node[1][1] + if t == 'for': + assignNode = self.com_assign(node[2], OP_ASSIGN) + compNode = self.com_node(node[4]) + newfor = ListCompFor(assignNode, compNode, []) + newfor.lineno = node[1][2] + fors.append(newfor) + if len(node) == 5: + node = None + elif type == 'list': + node = self.com_list_iter(node[5]) else: - raise SyntaxError, \ - ("unexpected list comprehension element: %s %d" - % (node, lineno)) - return ListComp(expr, fors, lineno=lineno) - - def com_list_iter(self, node): - assert node[0] == symbol.list_iter - return node[1] - else: - def com_list_constructor(self, nodelist): - values = [] - for i in range(1, len(nodelist), 2): - values.append(self.com_node(nodelist[i])) - return List(values, lineno=values[0].lineno) - - if hasattr(symbol, 'gen_for'): - def com_generator_expression(self, expr, node): - # gen_iter: gen_for | gen_if - # gen_for: 'for' exprlist 'in' test [gen_iter] - # gen_if: 'if' test [gen_iter] - - lineno = node[1][2] - fors = [] - while node: - t = node[1][1] - if t == 'for': - assignNode = self.com_assign(node[2], OP_ASSIGN) - genNode = self.com_node(node[4]) - newfor = GenExprFor(assignNode, genNode, [], - lineno=node[1][2]) - fors.append(newfor) - if (len(node)) == 5: - node = None - else: - node = self.com_gen_iter(node[5]) - elif t == 'if': - test = self.com_node(node[2]) - newif = GenExprIf(test, lineno=node[1][2]) - newfor.ifs.append(newif) - if len(node) == 3: - node = None - else: - node = self.com_gen_iter(node[3]) + node = self.com_comp_iter(node[5]) + elif t == 'if': + test = self.com_node(node[2]) + newif = ListCompIf(test, lineno=node[1][2]) + newfor.ifs.append(newif) + if len(node) == 3: + node = None + elif type == 'list': + node = self.com_list_iter(node[3]) else: - raise SyntaxError, \ - ("unexpected generator expression element: %s %d" - % (node, lineno)) - fors[0].is_outmost = True - return GenExpr(GenExprInner(expr, fors), lineno=lineno) - - def com_gen_iter(self, node): - assert node[0] == symbol.gen_iter - return node[1] - - def com_dictmaker(self, nodelist): - # dictmaker: test ':' test (',' test ':' value)* [','] - items = [] - for i in range(1, len(nodelist), 4): - items.append((self.com_node(nodelist[i]), - self.com_node(nodelist[i+2]))) - return Dict(items, lineno=items[0][0].lineno) + node = self.com_comp_iter(node[3]) + else: + raise SyntaxError, \ + ("unexpected comprehension element: %s %d" + % (node, lineno)) + if type == 'list': + return ListComp(expr1, fors, lineno=lineno) + elif type == 'set': + return SetComp(expr1, fors, lineno=lineno) + elif type == 'dict': + return DictComp(expr1, expr2, fors, lineno=lineno) + else: + raise ValueError("unexpected comprehension type: " + repr(type)) + + def com_list_iter(self, node): + assert node[0] == symbol.list_iter + return node[1] + + def com_comp_iter(self, node): + assert node[0] == symbol.comp_iter + return node[1] + + def com_generator_expression(self, expr, node): + # comp_iter: comp_for | comp_if + # comp_for: 'for' exprlist 'in' test [comp_iter] + # comp_if: 'if' test [comp_iter] + + lineno = node[1][2] + fors = [] + while node: + t = node[1][1] + if t == 'for': + assignNode = self.com_assign(node[2], OP_ASSIGN) + genNode = self.com_node(node[4]) + newfor = GenExprFor(assignNode, genNode, [], + lineno=node[1][2]) + fors.append(newfor) + if (len(node)) == 5: + node = None + else: + node = self.com_comp_iter(node[5]) + elif t == 'if': + test = self.com_node(node[2]) + newif = GenExprIf(test, lineno=node[1][2]) + newfor.ifs.append(newif) + if len(node) == 3: + node = None + else: + node = self.com_comp_iter(node[3]) + else: + raise SyntaxError, \ + ("unexpected generator expression element: %s %d" + % (node, lineno)) + fors[0].is_outmost = True + return GenExpr(GenExprInner(expr, fors), lineno=lineno) + + def com_dictorsetmaker(self, nodelist): + # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | + # (test (comp_for | (',' test)* [','])) ) + assert nodelist[0] == symbol.dictorsetmaker + nodelist = nodelist[1:] + if len(nodelist) == 1 or nodelist[1][0] == token.COMMA: + # set literal + items = [] + for i in range(0, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + return Set(items, lineno=items[0].lineno) + elif nodelist[1][0] == symbol.comp_for: + # set comprehension + expr = self.com_node(nodelist[0]) + return self.com_comprehension(expr, None, nodelist[1], 'set') + elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for: + # dict comprehension + assert nodelist[1][0] == token.COLON + key = self.com_node(nodelist[0]) + value = self.com_node(nodelist[2]) + return self.com_comprehension(key, value, nodelist[3], 'dict') + else: + # dict literal + items = [] + for i in range(0, len(nodelist), 4): + items.append((self.com_node(nodelist[i]), + self.com_node(nodelist[i+2]))) + return Dict(items, lineno=items[0][0].lineno) def com_apply_trailer(self, primaryNode, nodelist): t = nodelist[1][0] @@ -1245,7 +1281,7 @@ class Transformer: kw, result = self.com_argument(node, kw, star_node) if len_nodelist != 2 and isinstance(result, GenExpr) \ - and len(node) == 3 and node[2][0] == symbol.gen_for: + and len(node) == 3 and node[2][0] == symbol.comp_for: # allow f(x for x in y), but reject f(x for x in y, 1) # should use f((x for x in y), 1) instead of f(x for x in y, 1) raise SyntaxError, 'generator expression needs parenthesis' @@ -1257,7 +1293,7 @@ class Transformer: lineno=extractLineNo(nodelist)) def com_argument(self, nodelist, kw, star_node): - if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for: + if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for: test = self.com_node(nodelist[1]) return 0, self.com_generator_expression(test, nodelist[2]) if len(nodelist) == 2: |