From b17f27672208a07318f8aa62a1bd64b18e9961d1 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 24 Dec 2015 08:49:50 -0500
Subject: WIP: measure branches with ast instead of bytecode

--HG--
branch : ast-branch
---
 coverage/parser.py | 230 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 228 insertions(+), 2 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 7b8a60f1..fb2cf955 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -3,6 +3,7 @@
 
 """Code parsing for coverage.py."""
 
+import ast
 import collections
 import dis
 import re
@@ -260,6 +261,18 @@ class PythonParser(object):
                     self._all_arcs.add((fl1, fl2))
         return self._all_arcs
 
+    def ast_arcs(self):
+        aaa = AstArcAnalyzer(self.text)
+        arcs = aaa.collect_arcs()
+
+        arcs_ = set()
+        for l1, l2 in arcs:
+            fl1 = self.first_line(l1)
+            fl2 = self.first_line(l2)
+            if fl1 != fl2:
+                arcs_.add((fl1, fl2))
+        return arcs_
+
     def exit_counts(self):
         """Get a count of exits from that each line.
 
@@ -288,6 +301,168 @@ class PythonParser(object):
         return exit_counts
 
 
+class AstArcAnalyzer(object):
+    def __init__(self, text):
+        self.root_node = ast.parse(text)
+        ast_dump(self.root_node)
+
+        self.arcs = None
+        # References to the nearest enclosing thing of its kind.
+        self.function_start = None
+        self.loop_start = None
+
+        # Break-exits from a loop
+        self.break_exits = None
+
+    def line_for_node(self, node):
+        """What is the right line number to use for this node?"""
+        node_name = node.__class__.__name__
+        if node_name == "Assign":
+            return node.value.lineno
+        elif node_name == "comprehension":
+            # TODO: is this how to get the line number for a comprehension?
+            return node.target.lineno
+        else:
+            return node.lineno
+
+    def collect_arcs(self):
+        self.arcs = set()
+        self.add_arcs_for_code_objects(self.root_node)
+        return self.arcs
+
+    def add_arcs(self, node):
+        """add the arcs for `node`.
+
+        Return a set of line numbers, exits from this node to the next.
+        """
+        node_name = node.__class__.__name__
+        #print("Adding arcs for {}".format(node_name))
+
+        handler = getattr(self, "handle_" + node_name, self.handle_default)
+        return handler(node)
+
+    def add_body_arcs(self, body, from_line):
+        prev_lines = set([from_line])
+        for body_node in body:
+            lineno = self.line_for_node(body_node)
+            for prev_lineno in prev_lines:
+                self.arcs.add((prev_lineno, lineno))
+            prev_lines = self.add_arcs(body_node)
+        return prev_lines
+
+    def is_constant_expr(self, node):
+        """Is this a compile-time constant?"""
+        node_name = node.__class__.__name__
+        return node_name in ["NameConstant", "Num"]
+
+    # tests to write:
+    # TODO: while EXPR:
+    # TODO: while False:
+    # TODO: multi-target assignment with computed targets
+    # TODO: listcomps hidden deep in other expressions
+    # TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
+    # TODO: multi-line listcomps
+    # TODO: nested function definitions
+
+    def handle_Break(self, node):
+        here = self.line_for_node(node)
+        # TODO: what if self.break_exits is None?
+        self.break_exits.add(here)
+        return set([])
+
+    def handle_Continue(self, node):
+        here = self.line_for_node(node)
+        # TODO: what if self.loop_start is None?
+        self.arcs.add((here, self.loop_start))
+        return set([])
+
+    def handle_For(self, node):
+        start = self.line_for_node(node.iter)
+        loop_state = self.loop_start, self.break_exits
+        self.loop_start = start
+        self.break_exits = set()
+        exits = self.add_body_arcs(node.body, from_line=start)
+        for exit in exits:
+            self.arcs.add((exit, start))
+        exits = self.break_exits
+        self.loop_start, self.break_exits = loop_state
+        if node.orelse:
+            else_start = self.line_for_node(node.orelse[0])
+            self.arcs.add((start, else_start))
+            else_exits = self.add_body_arcs(node.orelse, from_line=start)
+            exits |= else_exits
+        else:
+            # no else clause: exit from the for line.
+            exits.add(start)
+        return exits
+
+    def handle_FunctionDef(self, node):
+        start = self.line_for_node(node)
+        # the body is handled in add_arcs_for_code_objects.
+        exits = set([start])
+        return exits
+
+    def handle_If(self, node):
+        start = self.line_for_node(node.test)
+        exits = self.add_body_arcs(node.body, from_line=start)
+        exits |= self.add_body_arcs(node.orelse, from_line=start)
+        return exits
+
+    def handle_Module(self, node):
+        raise Exception("TODO: this shouldn't happen")
+
+    def handle_Return(self, node):
+        here = self.line_for_node(node)
+        # TODO: what if self.function_start is None?
+        self.arcs.add((here, -self.function_start))
+        return set([])
+
+    def handle_While(self, node):
+        constant_test = self.is_constant_expr(node.test)
+        start = to_top = self.line_for_node(node.test)
+        if constant_test:
+            to_top = self.line_for_node(node.body[0])
+        loop_state = self.loop_start, self.break_exits
+        self.loop_start = start
+        self.break_exits = set()
+        exits = self.add_body_arcs(node.body, from_line=start)
+        for exit in exits:
+            self.arcs.add((exit, to_top))
+        exits = self.break_exits
+        self.loop_start, self.break_exits = loop_state
+        # TODO: orelse
+        return exits
+
+    def handle_default(self, node):
+        node_name = node.__class__.__name__
+        if node_name not in ["Assign", "Assert", "AugAssign", "Expr"]:
+            print("*** Unhandled: {}".format(node))
+        return set([self.line_for_node(node)])
+
+    def add_arcs_for_code_objects(self, root_node):
+        for node in ast.walk(root_node):
+            node_name = node.__class__.__name__
+            if node_name == "Module":
+                start = self.line_for_node(node.body[0])
+                exits = self.add_body_arcs(node.body, from_line=-1)
+                for exit in exits:
+                    self.arcs.add((exit, -start))
+            elif node_name == "FunctionDef":
+                start = self.line_for_node(node)
+                self.function_start = start
+                func_exits = self.add_body_arcs(node.body, from_line=-1)
+                for exit in func_exits:
+                    self.arcs.add((exit, -start))
+                self.function_start = None
+            elif node_name == "comprehension":
+                start = self.line_for_node(node)
+                self.arcs.add((-1, start))
+                self.arcs.add((start, -start))
+                # TODO: guaranteed this won't work for multi-line comps.
+
+
+
+
 ## Opcodes that guide the ByteParser.
 
 def _opcode(name):
@@ -321,7 +496,7 @@ OPS_CHUNK_BEGIN = _opcode_set('JUMP_ABSOLUTE', 'JUMP_FORWARD')
 
 # Opcodes that push a block on the block stack.
 OPS_PUSH_BLOCK = _opcode_set(
-    'SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH'
+    'SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH',
 )
 
 # Block types for exception handling.
@@ -330,6 +505,8 @@ OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY')
 # Opcodes that pop a block from the block stack.
 OPS_POP_BLOCK = _opcode_set('POP_BLOCK')
 
+OPS_GET_AITER = _opcode_set('GET_AITER')
+
 # Opcodes that have a jump destination, but aren't really a jump.
 OPS_NO_JUMP = OPS_PUSH_BLOCK
 
@@ -449,6 +626,8 @@ class ByteParser(object):
         # is a count of how many ignores are left.
         ignore_branch = 0
 
+        ignore_pop_block = 0
+
         # We have to handle the last two bytecodes specially.
         ult = penult = None
 
@@ -507,7 +686,10 @@ class ByteParser(object):
                 block_stack.append((bc.op, bc.jump_to))
             if bc.op in OPS_POP_BLOCK:
                 # The opcode pops a block from the block stack.
-                block_stack.pop()
+                if ignore_pop_block:
+                    ignore_pop_block -= 1
+                else:
+                    block_stack.pop()
             if bc.op in OPS_CHUNK_END:
                 # This opcode forces the end of the chunk.
                 if bc.op == OP_BREAK_LOOP:
@@ -527,6 +709,15 @@ class ByteParser(object):
                 # branch, so that except's don't count as branches.
                 ignore_branch += 1
 
+            if bc.op in OPS_GET_AITER:
+                # GET_AITER is weird: First, it seems to generate one more
+                # POP_BLOCK than SETUP_*, so we have to prepare to ignore one
+                # of the POP_BLOCKS.  Second, we don't have a clear branch to
+                # the exit of the loop, so we peek into the block stack to find
+                # it.
+                ignore_pop_block += 1
+                chunk.exits.add(block_stack[-1][1])
+
             penult = ult
             ult = bc
 
@@ -686,3 +877,38 @@ class Chunk(object):
             "v" if self.entrance else "",
             list(self.exits),
         )
+
+
+SKIP_FIELDS = ["ctx"]
+
+def ast_dump(node, depth=0):
+    indent = " " * depth
+    lineno = getattr(node, "lineno", None)
+    if lineno is not None:
+        linemark = " @ {0}".format(lineno)
+    else:
+        linemark = ""
+    print("{0}<{1}{2}".format(indent, node.__class__.__name__, linemark))
+
+    indent += "    "
+    for field_name, value in ast.iter_fields(node):
+        if field_name in SKIP_FIELDS:
+            continue
+        prefix = "{0}{1}:".format(indent, field_name)
+        if value is None:
+            print("{0} None".format(prefix))
+        elif isinstance(value, (str, int)):
+            print("{0} {1!r}".format(prefix, value))
+        elif isinstance(value, list):
+            if value == []:
+                print("{0} []".format(prefix))
+            else:
+                print("{0} [".format(prefix))
+                for n in value:
+                    ast_dump(n, depth + 8)
+                print("{0}]".format(indent))
+        else:
+            print(prefix)
+            ast_dump(value, depth + 8)
+
+    print("{0}>".format(" " * depth))
-- 
cgit v1.2.1


From b7a35186425cfef265548afc75b527752bed0c9a Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 24 Dec 2015 19:46:00 -0500
Subject: A start on try/except/finally

--HG--
branch : ast-branch
---
 coverage/parser.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index fb2cf955..4b920f10 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -341,8 +341,9 @@ class AstArcAnalyzer(object):
         handler = getattr(self, "handle_" + node_name, self.handle_default)
         return handler(node)
 
-    def add_body_arcs(self, body, from_line):
-        prev_lines = set([from_line])
+    def add_body_arcs(self, body, from_line=None, prev_lines=None):
+        if prev_lines is None:
+            prev_lines = set([from_line])
         for body_node in body:
             lineno = self.line_for_node(body_node)
             for prev_lineno in prev_lines:
@@ -363,6 +364,7 @@ class AstArcAnalyzer(object):
     # TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
     # TODO: multi-line listcomps
     # TODO: nested function definitions
+    # TODO: multiple `except` clauses
 
     def handle_Break(self, node):
         here = self.line_for_node(node)
@@ -411,12 +413,30 @@ class AstArcAnalyzer(object):
     def handle_Module(self, node):
         raise Exception("TODO: this shouldn't happen")
 
+    def handle_Raise(self, node):
+        # `raise` statement jumps away, no exits from here.
+        return set([])
+
     def handle_Return(self, node):
         here = self.line_for_node(node)
         # TODO: what if self.function_start is None?
         self.arcs.add((here, -self.function_start))
         return set([])
 
+    def handle_Try(self, node):
+        start = self.line_for_node(node)
+        exits = self.add_body_arcs(node.body, from_line=start)
+        handler_exits = set()
+        for handler_node in node.handlers:
+            handler_start = self.line_for_node(handler_node)
+            # TODO: handler_node.name and handler_node.type
+            handler_exits |= self.add_body_arcs(handler_node.body, from_line=handler_start)
+        # TODO: node.orelse
+        # TODO: node.finalbody
+        if node.finalbody:
+            exits = self.add_body_arcs(node.finalbody, prev_lines=exits|handler_exits)
+        return exits
+
     def handle_While(self, node):
         constant_test = self.is_constant_expr(node.test)
         start = to_top = self.line_for_node(node.test)
-- 
cgit v1.2.1


From 35c09545a39e70065ce55264f2688ac87dd6a725 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Mon, 28 Dec 2015 16:48:05 -0500
Subject: Execution flows from the end of exception handlers to the finally

--HG--
branch : ast-branch
---
 coverage/parser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 4b920f10..65b1f0fb 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -432,9 +432,9 @@ class AstArcAnalyzer(object):
             # TODO: handler_node.name and handler_node.type
             handler_exits |= self.add_body_arcs(handler_node.body, from_line=handler_start)
         # TODO: node.orelse
-        # TODO: node.finalbody
+        exits |= handler_exits
         if node.finalbody:
-            exits = self.add_body_arcs(node.finalbody, prev_lines=exits|handler_exits)
+            exits = self.add_body_arcs(node.finalbody, prev_lines=exits)
         return exits
 
     def handle_While(self, node):
-- 
cgit v1.2.1


From 4b33f09a3d46e5dd051d060a1926567fd418cbb7 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 31 Dec 2015 15:39:30 -0500
Subject: Exception tests pass on py3

--HG--
branch : ast-branch
---
 coverage/parser.py | 143 ++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 113 insertions(+), 30 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 65b1f0fb..ff2d2bec 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -11,7 +11,7 @@ import token
 import tokenize
 
 from coverage.backward import range    # pylint: disable=redefined-builtin
-from coverage.backward import bytes_to_ints
+from coverage.backward import bytes_to_ints, string_class
 from coverage.bytecode import ByteCodes, CodeObjects
 from coverage.misc import contract, nice_pair, join_regex
 from coverage.misc import CoverageException, NoSource, NotPython
@@ -245,7 +245,7 @@ class PythonParser(object):
         starts = self.raw_statements - ignore
         self.statements = self.first_lines(starts) - ignore
 
-    def arcs(self):
+    def old_arcs(self):
         """Get information about the arcs available in the code.
 
         Returns a set of line number pairs.  Line numbers have been normalized
@@ -261,7 +261,7 @@ class PythonParser(object):
                     self._all_arcs.add((fl1, fl2))
         return self._all_arcs
 
-    def ast_arcs(self):
+    def arcs(self):
         aaa = AstArcAnalyzer(self.text)
         arcs = aaa.collect_arcs()
 
@@ -301,18 +301,36 @@ class PythonParser(object):
         return exit_counts
 
 
+class LoopBlock(object):
+    def __init__(self, start):
+        self.start = start
+        self.break_exits = set()
+
+class FunctionBlock(object):
+    def __init__(self, start):
+        self.start = start
+
+class TryBlock(object):
+    def __init__(self, handler_start=None, final_start=None):
+        self.handler_start = handler_start  # TODO: is this used?
+        self.final_start = final_start      # TODO: is this used?
+        self.break_from = set([])
+        self.continue_from = set([])
+        self.return_from = set([])
+        self.raise_from = set([])
+
+
 class AstArcAnalyzer(object):
     def __init__(self, text):
         self.root_node = ast.parse(text)
-        ast_dump(self.root_node)
+        #ast_dump(self.root_node)
 
         self.arcs = None
-        # References to the nearest enclosing thing of its kind.
-        self.function_start = None
-        self.loop_start = None
+        self.block_stack = []
 
-        # Break-exits from a loop
-        self.break_exits = None
+    def blocks(self):
+        """Yield the blocks in nearest-to-farthest order."""
+        return reversed(self.block_stack)
 
     def line_for_node(self, node):
         """What is the right line number to use for this node?"""
@@ -366,28 +384,70 @@ class AstArcAnalyzer(object):
     # TODO: nested function definitions
     # TODO: multiple `except` clauses
 
+    def process_break_exits(self, exits):
+        for block in self.blocks():
+            if isinstance(block, LoopBlock):
+                # TODO: what if there is no loop?
+                block.break_exits.update(exits)
+                break
+            elif isinstance(block, TryBlock) and block.final_start:
+                block.break_from.update(exits)
+                break
+
+    def process_continue_exits(self, exits):
+        for block in self.blocks():
+            if isinstance(block, LoopBlock):
+                # TODO: what if there is no loop?
+                for exit in exits:
+                    self.arcs.add((exit, block.start))
+                break
+            elif isinstance(block, TryBlock) and block.final_start:
+                block.continue_from.update(exits)
+                break
+
+    def process_raise_exits(self, exits):
+        for block in self.blocks():
+            if isinstance(block, TryBlock):
+                if block.handler_start:
+                    for exit in exits:
+                        self.arcs.add((exit, block.handler_start))
+                    break
+                elif block.final_start:
+                    block.raise_from.update(exits)
+                    break
+            elif isinstance(block, FunctionBlock):
+                for exit in exits:
+                    self.arcs.add((exit, -block.start))
+                break
+
+    def process_return_exits(self, exits):
+        for block in self.blocks():
+            if isinstance(block, FunctionBlock):
+                # TODO: what if there is no enclosing function?
+                for exit in exits:
+                    self.arcs.add((exit, -block.start))
+                break
+
+    ## Handlers
+
     def handle_Break(self, node):
         here = self.line_for_node(node)
-        # TODO: what if self.break_exits is None?
-        self.break_exits.add(here)
+        self.process_break_exits([here])
         return set([])
 
     def handle_Continue(self, node):
         here = self.line_for_node(node)
-        # TODO: what if self.loop_start is None?
-        self.arcs.add((here, self.loop_start))
+        self.process_continue_exits([here])
         return set([])
 
     def handle_For(self, node):
         start = self.line_for_node(node.iter)
-        loop_state = self.loop_start, self.break_exits
-        self.loop_start = start
-        self.break_exits = set()
+        self.block_stack.append(LoopBlock(start=start))
         exits = self.add_body_arcs(node.body, from_line=start)
         for exit in exits:
             self.arcs.add((exit, start))
-        exits = self.break_exits
-        self.loop_start, self.break_exits = loop_state
+        my_block = self.block_stack.pop()
+        exits = my_block.break_exits
         if node.orelse:
             else_start = self.line_for_node(node.orelse[0])
             self.arcs.add((start, else_start))
@@ -415,15 +475,29 @@ class AstArcAnalyzer(object):
 
     def handle_Raise(self, node):
         # `raise` statement jumps away, no exits from here.
+        here = self.line_for_node(node)
+        self.process_raise_exits([here])
         return set([])
 
     def handle_Return(self, node):
+        # TODO: deal with returning through a finally.
         here = self.line_for_node(node)
-        # TODO: what if self.function_start is None?
-        self.arcs.add((here, -self.function_start))
+        self.process_return_exits([here])
         return set([])
 
     def handle_Try(self, node):
+        # try/finally is tricky. If there's a finally clause, then we need a
+        # FinallyBlock to track what flows might go through the finally instead
+        # of their normal flow.
+        if node.handlers:
+            handler_start = self.line_for_node(node.handlers[0])
+        else:
+            handler_start = None
+        if node.finalbody:
+            final_start = self.line_for_node(node.finalbody[0])
+        else:
+            final_start = None
+        self.block_stack.append(TryBlock(handler_start=handler_start, final_start=final_start))
         start = self.line_for_node(node)
         exits = self.add_body_arcs(node.body, from_line=start)
         handler_exits = set()
@@ -434,7 +508,17 @@ class AstArcAnalyzer(object):
         # TODO: node.orelse
         exits |= handler_exits
         if node.finalbody:
-            exits = self.add_body_arcs(node.finalbody, prev_lines=exits)
+            final_block = self.block_stack.pop()
+            final_from = exits | final_block.break_from | final_block.continue_from | final_block.raise_from | final_block.return_from
+            exits = self.add_body_arcs(node.finalbody, prev_lines=final_from)
+            if final_block.break_from:
+                self.process_break_exits(exits)
+            if final_block.continue_from:
+                self.process_continue_exits(exits)
+            if final_block.raise_from:
+                self.process_raise_exits(exits)
+            if final_block.return_from:
+                self.process_return_exits(exits)
         return exits
 
     def handle_While(self, node):
@@ -442,20 +526,19 @@ class AstArcAnalyzer(object):
         start = to_top = self.line_for_node(node.test)
         if constant_test:
             to_top = self.line_for_node(node.body[0])
-        loop_state = self.loop_start, self.break_exits
-        self.loop_start = start
-        self.break_exits = set()
+        self.block_stack.append(LoopBlock(start=start))
         exits = self.add_body_arcs(node.body, from_line=start)
         for exit in exits:
             self.arcs.add((exit, to_top))
-        exits = self.break_exits
-        self.loop_start, self.break_exits = loop_state
+        # TODO: while loop that finishes?
+        my_block = self.block_stack.pop()
+        exits = my_block.break_exits
         # TODO: orelse
         return exits
 
     def handle_default(self, node):
         node_name = node.__class__.__name__
-        if node_name not in ["Assign", "Assert", "AugAssign", "Expr"]:
+        if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Pass"]:
             print("*** Unhandled: {}".format(node))
         return set([self.line_for_node(node)])
 
@@ -469,11 +552,11 @@ class AstArcAnalyzer(object):
                     self.arcs.add((exit, -start))
             elif node_name == "FunctionDef":
                 start = self.line_for_node(node)
-                self.function_start = start
+                self.block_stack.append(FunctionBlock(start=start))
                 func_exits = self.add_body_arcs(node.body, from_line=-1)
+                self.block_stack.pop()
                 for exit in func_exits:
                     self.arcs.add((exit, -start))
-                self.function_start = None
             elif node_name == "comprehension":
                 start = self.line_for_node(node)
                 self.arcs.add((-1, start))
@@ -917,7 +1000,7 @@ def ast_dump(node, depth=0):
         prefix = "{0}{1}:".format(indent, field_name)
         if value is None:
             print("{0} None".format(prefix))
-        elif isinstance(value, (str, int)):
+        elif isinstance(value, (string_class, int, float)):
             print("{0} {1!r}".format(prefix, value))
         elif isinstance(value, list):
             if value == []:
-- 
cgit v1.2.1


From 5a6627ce5050d331095c4b03aed8e540f3ed651f Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 31 Dec 2015 16:20:09 -0500
Subject: Make other comprehensions work on py2 and py3

--HG--
branch : ast-branch
---
 coverage/parser.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index ff2d2bec..36fa729c 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -10,6 +10,7 @@ import re
 import token
 import tokenize
 
+from coverage import env
 from coverage.backward import range    # pylint: disable=redefined-builtin
 from coverage.backward import bytes_to_ints, string_class
 from coverage.bytecode import ByteCodes, CodeObjects
@@ -323,7 +324,7 @@ class TryBlock(object):
 class AstArcAnalyzer(object):
     def __init__(self, text):
         self.root_node = ast.parse(text)
-        #ast_dump(self.root_node)
+        ast_dump(self.root_node)
 
         self.arcs = None
         self.block_stack = []
@@ -542,6 +543,10 @@ class AstArcAnalyzer(object):
             print("*** Unhandled: {}".format(node))
         return set([self.line_for_node(node)])
 
+    CODE_COMPREHENSIONS = set(["GeneratorExp", "DictComp", "SetComp"])
+    if env.PY3:
+        CODE_COMPREHENSIONS.add("ListComp")
+
     def add_arcs_for_code_objects(self, root_node):
         for node in ast.walk(root_node):
             node_name = node.__class__.__name__
@@ -557,13 +562,12 @@ class AstArcAnalyzer(object):
                 self.block_stack.pop()
                 for exit in func_exits:
                     self.arcs.add((exit, -start))
-            elif node_name == "comprehension":
-                start = self.line_for_node(node)
-                self.arcs.add((-1, start))
-                self.arcs.add((start, -start))
-                # TODO: guaranteed this won't work for multi-line comps.
-
-
+            elif node_name in self.CODE_COMPREHENSIONS:
+                for gen in node.generators:
+                    start = self.line_for_node(gen)
+                    self.arcs.add((-1, start))
+                    self.arcs.add((start, -start))
+                    # TODO: guaranteed this won't work for multi-line comps.
 
 
 ## Opcodes that guide the ByteParser.
-- 
cgit v1.2.1


From f5acc8c5651287022e5b7d7d98e1be9393674c47 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 31 Dec 2015 16:39:17 -0500
Subject: Support exception arcs on py2, where the ast still has separate
 TryExcept and TryFinally nodes

--HG--
branch : ast-branch
---
 coverage/parser.py | 44 +++++++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 17 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 36fa729c..d8b0beea 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -487,38 +487,48 @@ class AstArcAnalyzer(object):
         return set([])
 
     def handle_Try(self, node):
+        return self.try_work(node, node.body, node.handlers, node.orelse, node.finalbody)
+
+    def handle_TryExcept(self, node):
+        return self.try_work(node, node.body, node.handlers, node.orelse, None)
+
+    def handle_TryFinally(self, node):
+        return self.try_work(node, node.body, None, None, node.finalbody)
+
+    def try_work(self, node, body, handlers, orelse, finalbody):
         # try/finally is tricky. If there's a finally clause, then we need a
         # FinallyBlock to track what flows might go through the finally instead
         # of their normal flow.
-        if node.handlers:
-            handler_start = self.line_for_node(node.handlers[0])
+        if handlers:
+            handler_start = self.line_for_node(handlers[0])
         else:
             handler_start = None
-        if node.finalbody:
-            final_start = self.line_for_node(node.finalbody[0])
+        if finalbody:
+            final_start = self.line_for_node(finalbody[0])
         else:
             final_start = None
         self.block_stack.append(TryBlock(handler_start=handler_start, final_start=final_start))
         start = self.line_for_node(node)
-        exits = self.add_body_arcs(node.body, from_line=start)
+        exits = self.add_body_arcs(body, from_line=start)
+        try_block = self.block_stack.pop()
         handler_exits = set()
-        for handler_node in node.handlers:
-            handler_start = self.line_for_node(handler_node)
-            # TODO: handler_node.name and handler_node.type
-            handler_exits |= self.add_body_arcs(handler_node.body, from_line=handler_start)
+        if handlers:
+            for handler_node in handlers:
+                handler_start = self.line_for_node(handler_node)
+                # TODO: handler_node.name and handler_node.type
+                handler_exits |= self.add_body_arcs(handler_node.body, from_line=handler_start)
         # TODO: node.orelse
         exits |= handler_exits
-        if node.finalbody:
-            final_block = self.block_stack.pop()
-            final_from = exits | final_block.break_from | final_block.continue_from | final_block.raise_from | final_block.return_from
-            exits = self.add_body_arcs(node.finalbody, prev_lines=final_from)
-            if final_block.break_from:
+        if finalbody:
+            final_from = exits | try_block.break_from | try_block.continue_from | try_block.raise_from | try_block.return_from
+            exits = self.add_body_arcs(finalbody, prev_lines=final_from)
+            if try_block.break_from:
                 self.process_break_exits(exits)
-            if final_block.continue_from:
+            if try_block.continue_from:
                 self.process_continue_exits(exits)
-            if final_block.raise_from:
+            if try_block.raise_from:
                 self.process_raise_exits(exits)
-            if final_block.return_from:
+            if try_block.return_from:
                 self.process_return_exits(exits)
         return exits
 
-- 
cgit v1.2.1


From 704fa07b52715720da0f7b2d264ea41fce7441e8 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 31 Dec 2015 18:24:36 -0500
Subject: Support classdef and some async keywords

--HG--
branch : ast-branch
---
 coverage/parser.py | 58 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 17 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index d8b0beea..d599bef9 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -6,6 +6,7 @@
 import ast
 import collections
 import dis
+import os
 import re
 import token
 import tokenize
@@ -315,16 +316,17 @@ class TryBlock(object):
     def __init__(self, handler_start=None, final_start=None):
         self.handler_start = handler_start  # TODO: is this used?
         self.final_start = final_start      # TODO: is this used?
-        self.break_from = set([])
-        self.continue_from = set([])
-        self.return_from = set([])
-        self.raise_from = set([])
+        self.break_from = set()
+        self.continue_from = set()
+        self.return_from = set()
+        self.raise_from = set()
 
 
 class AstArcAnalyzer(object):
     def __init__(self, text):
         self.root_node = ast.parse(text)
-        ast_dump(self.root_node)
+        if int(os.environ.get("COVERAGE_ASTDUMP", 0)):
+            ast_dump(self.root_node)
 
         self.arcs = None
         self.block_stack = []
@@ -434,12 +436,17 @@ class AstArcAnalyzer(object):
     def handle_Break(self, node):
         here = self.line_for_node(node)
         self.process_break_exits([here])
-        return set([])
+        return set()
+
+    def handle_ClassDef(self, node):
+        start = self.line_for_node(node)
+        # the body is handled in add_arcs_for_code_objects.
+        return set([start])
 
     def handle_Continue(self, node):
         here = self.line_for_node(node)
         self.process_continue_exits([here])
-        return set([])
+        return set()
 
     def handle_For(self, node):
         start = self.line_for_node(node.iter)
@@ -459,11 +466,14 @@ class AstArcAnalyzer(object):
             exits.add(start)
         return exits
 
+    handle_AsyncFor = handle_For
+
     def handle_FunctionDef(self, node):
         start = self.line_for_node(node)
         # the body is handled in add_arcs_for_code_objects.
-        exits = set([start])
-        return exits
+        return set([start])
+
+    handle_AsyncFunctionDef = handle_FunctionDef
 
     def handle_If(self, node):
         start = self.line_for_node(node.test)
@@ -478,13 +488,13 @@ class AstArcAnalyzer(object):
         # `raise` statement jumps away, no exits from here.
         here = self.line_for_node(node)
         self.process_raise_exits([here])
-        return set([])
+        return set()
 
     def handle_Return(self, node):
         # TODO: deal with returning through a finally.
         here = self.line_for_node(node)
         self.process_return_exits([here])
-        return set([])
+        return set()
 
     def handle_Try(self, node):
         return self.try_work(node, node.body, node.handlers, node.orelse, node.finalbody)
@@ -541,15 +551,17 @@ class AstArcAnalyzer(object):
         exits = self.add_body_arcs(node.body, from_line=start)
         for exit in exits:
             self.arcs.add((exit, to_top))
-        # TODO: while loop that finishes?
+        exits = set()
+        if not constant_test:
+            exits.add(start)
         my_block = self.block_stack.pop()
-        exits = my_block.break_exits
+        exits.update(my_block.break_exits)
         # TODO: orelse
         return exits
 
     def handle_default(self, node):
         node_name = node.__class__.__name__
-        if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Pass"]:
+        if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Import", "Pass", "Print"]:
             print("*** Unhandled: {}".format(node))
         return set([self.line_for_node(node)])
 
@@ -565,19 +577,31 @@ class AstArcAnalyzer(object):
                 exits = self.add_body_arcs(node.body, from_line=-1)
                 for exit in exits:
                     self.arcs.add((exit, -start))
-            elif node_name == "FunctionDef":
+            elif node_name in ["FunctionDef", "AsyncFunctionDef"]:
                 start = self.line_for_node(node)
                 self.block_stack.append(FunctionBlock(start=start))
-                func_exits = self.add_body_arcs(node.body, from_line=-1)
+                exits = self.add_body_arcs(node.body, from_line=-1)
                 self.block_stack.pop()
-                for exit in func_exits:
+                for exit in exits:
+                    self.arcs.add((exit, -start))
+            elif node_name == "ClassDef":
+                start = self.line_for_node(node)
+                self.arcs.add((-1, start))
+                exits = self.add_body_arcs(node.body, from_line=start)
+                for exit in exits:
                     self.arcs.add((exit, -start))
             elif node_name in self.CODE_COMPREHENSIONS:
+                # TODO: tests for when generators is more than one?
                 for gen in node.generators:
                     start = self.line_for_node(gen)
                     self.arcs.add((-1, start))
                     self.arcs.add((start, -start))
                     # TODO: guaranteed this won't work for multi-line comps.
+            elif node_name == "Lambda":
+                start = self.line_for_node(node)
+                self.arcs.add((-1, start))
+                self.arcs.add((start, -start))
+                # TODO: test multi-line lambdas
 
 
 ## Opcodes that guide the ByteParser.
-- 
cgit v1.2.1


From 334f95902f91e54e60600072d7e1816670627718 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 1 Jan 2016 10:53:45 -0500
Subject: Support 'with'

--HG--
branch : ast-branch
---
 coverage/parser.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index d599bef9..a5e12d35 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -264,16 +264,17 @@ class PythonParser(object):
         return self._all_arcs
 
     def arcs(self):
-        aaa = AstArcAnalyzer(self.text)
-        arcs = aaa.collect_arcs()
+        if self._all_arcs is None:
+            aaa = AstArcAnalyzer(self.text)
+            arcs = aaa.collect_arcs()
 
-        arcs_ = set()
-        for l1, l2 in arcs:
-            fl1 = self.first_line(l1)
-            fl2 = self.first_line(l2)
-            if fl1 != fl2:
-                arcs_.add((fl1, fl2))
-        return arcs_
+            self._all_arcs = set()
+            for l1, l2 in arcs:
+                fl1 = self.first_line(l1)
+                fl2 = self.first_line(l2)
+                if fl1 != fl2:
+                    self._all_arcs.add((fl1, fl2))
+        return self._all_arcs
 
     def exit_counts(self):
         """Get a count of exits from that each line.
@@ -559,6 +560,13 @@ class AstArcAnalyzer(object):
         # TODO: orelse
         return exits
 
+    def handle_With(self, node):
+        start = self.line_for_node(node)
+        exits = self.add_body_arcs(node.body, from_line=start)
+        return exits
+
+    handle_AsyncWith = handle_With
+
     def handle_default(self, node):
         node_name = node.__class__.__name__
         if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Import", "Pass", "Print"]:
-- 
cgit v1.2.1


From 9edd625b8fdb09b5494471d460eba11148104e28 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 1 Jan 2016 12:18:57 -0500
Subject: All test_arcs.py tests pass on py27 and py35

--HG--
branch : ast-branch
---
 coverage/parser.py | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index a5e12d35..b2618921 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -339,13 +339,27 @@ class AstArcAnalyzer(object):
     def line_for_node(self, node):
         """What is the right line number to use for this node?"""
         node_name = node.__class__.__name__
-        if node_name == "Assign":
-            return node.value.lineno
-        elif node_name == "comprehension":
-            # TODO: is this how to get the line number for a comprehension?
-            return node.target.lineno
-        else:
-            return node.lineno
+        handler = getattr(self, "line_" + node_name, self.line_default)
+        return handler(node)
+
+    def line_Assign(self, node):
+        return self.line_for_node(node.value)
+
+    def line_Dict(self, node):
+        # Python 3.5 changed how dict literals are made.
+        if env.PYVERSION >= (3, 5):
+            return node.keys[0].lineno
+        return node.lineno
+
+    def line_List(self, node):
+        return self.line_for_node(node.elts[0])
+
+    def line_comprehension(self, node):
+        # TODO: is this how to get the line number for a comprehension?
+        return node.target.lineno
+
+    def line_default(self, node):
+        return node.lineno
 
     def collect_arcs(self):
         self.arcs = set()
@@ -358,8 +372,6 @@ class AstArcAnalyzer(object):
         Return a set of line numbers, exits from this node to the next.
         """
         node_name = node.__class__.__name__
-        #print("Adding arcs for {}".format(node_name))
-
         handler = getattr(self, "handle_" + node_name, self.handle_default)
         return handler(node)
 
-- 
cgit v1.2.1


From d6e221c8058259460cadfe62d5ca1bb0d93822cc Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 1 Jan 2016 13:38:06 -0500
Subject: test_arcs now passes for all Python versions

--HG--
branch : ast-branch
---
 coverage/parser.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index b2618921..33480924 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -388,7 +388,12 @@ class AstArcAnalyzer(object):
     def is_constant_expr(self, node):
         """Is this a compile-time constant?"""
         node_name = node.__class__.__name__
-        return node_name in ["NameConstant", "Num"]
+        if node_name in ["NameConstant", "Num"]:
+            return True
+        elif node_name == "Name":
+            if env.PY3 and node.id in ["True", "False", "None"]:
+                return True
+        return False
 
     # tests to write:
     # TODO: while EXPR:
-- 
cgit v1.2.1


From f1e583f91035983237d248b417b8ca9831ceac39 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 1 Jan 2016 16:10:50 -0500
Subject: check_coverage now assumes empty missing and unpredicted, and uses
 branch always

--HG--
branch : ast-branch
---
 coverage/parser.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 33480924..2396fb8c 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -347,7 +347,7 @@ class AstArcAnalyzer(object):
 
     def line_Dict(self, node):
         # Python 3.5 changed how dict literals are made.
-        if env.PYVERSION >= (3, 5):
+        if env.PYVERSION >= (3, 5) and node.keys:
             return node.keys[0].lineno
         return node.lineno
 
@@ -587,7 +587,7 @@ class AstArcAnalyzer(object):
     def handle_default(self, node):
         node_name = node.__class__.__name__
         if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Import", "Pass", "Print"]:
-            print("*** Unhandled: {}".format(node))
+            print("*** Unhandled: {0}".format(node))
         return set([self.line_for_node(node)])
 
     CODE_COMPREHENSIONS = set(["GeneratorExp", "DictComp", "SetComp"])
@@ -1049,6 +1049,10 @@ SKIP_FIELDS = ["ctx"]
 
 def ast_dump(node, depth=0):
     indent = " " * depth
+    if not isinstance(node, ast.AST):
+        print("{0}<{1} {2!r}>".format(indent, node.__class__.__name__, node))
+        return
+
     lineno = getattr(node, "lineno", None)
     if lineno is not None:
         linemark = " @ {0}".format(lineno)
-- 
cgit v1.2.1


From 6eb046c5937b9c78dab3451fae9348c4c721d6f9 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 2 Jan 2016 10:18:04 -0500
Subject: Handle yield-from and await.  All tests pass

--HG--
branch : ast-branch
---
 coverage/parser.py | 88 +++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 60 insertions(+), 28 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 2396fb8c..0462802e 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -327,11 +327,17 @@ class AstArcAnalyzer(object):
     def __init__(self, text):
         self.root_node = ast.parse(text)
         if int(os.environ.get("COVERAGE_ASTDUMP", 0)):
+            # Dump the AST so that failing tests have helpful output.
             ast_dump(self.root_node)
 
         self.arcs = None
         self.block_stack = []
 
+    def collect_arcs(self):
+        self.arcs = set()
+        self.add_arcs_for_code_objects(self.root_node)
+        return self.arcs
+
     def blocks(self):
         """Yield the blocks in nearest-to-farthest order."""
         return reversed(self.block_stack)
@@ -361,16 +367,19 @@ class AstArcAnalyzer(object):
     def line_default(self, node):
         return node.lineno
 
-    def collect_arcs(self):
-        self.arcs = set()
-        self.add_arcs_for_code_objects(self.root_node)
-        return self.arcs
-
     def add_arcs(self, node):
-        """add the arcs for `node`.
+        """Add the arcs for `node`.
 
         Return a set of line numbers, exits from this node to the next.
         """
+        # Yield-froms and awaits can appear anywhere.
+        # TODO: this is probably over-doing it, and too expensive. Can we
+        # instrument the ast walking to see how many nodes we are revisiting?
+        if isinstance(node, ast.stmt):
+            for name, value in ast.iter_fields(node):
+                if isinstance(value, ast.expr) and self.contains_return_expression(value):
+                    self.process_return_exits([self.line_for_node(node)])
+                    break
         node_name = node.__class__.__name__
         handler = getattr(self, "handle_" + node_name, self.handle_default)
         return handler(node)
@@ -404,6 +413,7 @@ class AstArcAnalyzer(object):
     # TODO: multi-line listcomps
     # TODO: nested function definitions
     # TODO: multiple `except` clauses
+    # TODO: return->finally
 
     def process_break_exits(self, exits):
         for block in self.blocks():
@@ -443,6 +453,7 @@ class AstArcAnalyzer(object):
 
     def process_return_exits(self, exits):
         for block in self.blocks():
+            # TODO: need a check here for TryBlock
             if isinstance(block, FunctionBlock):
                 # TODO: what if there is no enclosing function?
                 for exit in exits:
@@ -587,6 +598,7 @@ class AstArcAnalyzer(object):
     def handle_default(self, node):
         node_name = node.__class__.__name__
         if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Import", "Pass", "Print"]:
+            # TODO: put 1/0 here to find unhandled nodes.
             print("*** Unhandled: {0}".format(node))
         return set([self.line_for_node(node)])
 
@@ -628,6 +640,14 @@ class AstArcAnalyzer(object):
                 self.arcs.add((start, -start))
                 # TODO: test multi-line lambdas
 
+    def contains_return_expression(self, node):
+        """Is there a yield-from or await in `node` someplace?"""
+        for child in ast.walk(node):
+            if child.__class__.__name__ in ["YieldFrom", "Await"]:
+                return True
+
+        return False
+
 
 ## Opcodes that guide the ByteParser.
 
@@ -1045,7 +1065,13 @@ class Chunk(object):
         )
 
 
-SKIP_FIELDS = ["ctx"]
+SKIP_DUMP_FIELDS = ["ctx"]
+
+def is_simple_value(value):
+    return (
+        value in [None, [], (), {}, set()] or
+        isinstance(value, (string_class, int, float))
+    )
 
 def ast_dump(node, depth=0):
     indent = " " * depth
@@ -1055,30 +1081,36 @@ def ast_dump(node, depth=0):
 
     lineno = getattr(node, "lineno", None)
     if lineno is not None:
-        linemark = " @ {0}".format(lineno)
+        linemark = " @ {0}".format(node.lineno)
     else:
         linemark = ""
-    print("{0}<{1}{2}".format(indent, node.__class__.__name__, linemark))
-
-    indent += "    "
-    for field_name, value in ast.iter_fields(node):
-        if field_name in SKIP_FIELDS:
-            continue
-        prefix = "{0}{1}:".format(indent, field_name)
-        if value is None:
-            print("{0} None".format(prefix))
-        elif isinstance(value, (string_class, int, float)):
-            print("{0} {1!r}".format(prefix, value))
-        elif isinstance(value, list):
-            if value == []:
-                print("{0} []".format(prefix))
-            else:
+    head = "{0}<{1}{2}".format(indent, node.__class__.__name__, linemark)
+
+    named_fields = [
+        (name, value)
+        for name, value in ast.iter_fields(node)
+        if name not in SKIP_DUMP_FIELDS
+    ]
+    if not named_fields:
+        print("{0}>".format(head))
+    elif len(named_fields) == 1 and is_simple_value(named_fields[0][1]):
+        field_name, value = named_fields[0]
+        print("{0} {1}: {2!r}>".format(head, field_name, value))
+    else:
+        print(head)
+        print("{0}# mro: {1}".format(indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:])))
+        next_indent = indent + "    "
+        for field_name, value in named_fields:
+            prefix = "{0}{1}:".format(next_indent, field_name)
+            if is_simple_value(value):
+                print("{0} {1!r}".format(prefix, value))
+            elif isinstance(value, list):
                 print("{0} [".format(prefix))
                 for n in value:
                     ast_dump(n, depth + 8)
-                print("{0}]".format(indent))
-        else:
-            print(prefix)
-            ast_dump(value, depth + 8)
+                print("{0}]".format(next_indent))
+            else:
+                print(prefix)
+                ast_dump(value, depth + 8)
 
-    print("{0}>".format(" " * depth))
+        print("{0}>".format(indent))
-- 
cgit v1.2.1


From fa02e8b1d5f985c468d9c15869e092394298a41b Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 2 Jan 2016 11:09:10 -0500
Subject: Deal with a few more cases the test suite didn't turn up

--HG--
branch : ast-branch
---
 coverage/parser.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 0462802e..fc631fcc 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -355,10 +355,23 @@ class AstArcAnalyzer(object):
         # Python 3.5 changed how dict literals are made.
         if env.PYVERSION >= (3, 5) and node.keys:
             return node.keys[0].lineno
-        return node.lineno
+        else:
+            return node.lineno
 
     def line_List(self, node):
-        return self.line_for_node(node.elts[0])
+        if node.elts:
+            return self.line_for_node(node.elts[0])
+        else:
+            # TODO: test case for this branch:  x = []
+            return node.lineno
+
+    def line_Module(self, node):
+        if node.body:
+            return self.line_for_node(node.body[0])
+        else:
+            # Modules have no line number, they always start at 1.
+            # TODO: test case for empty module.
+            return 1
 
     def line_comprehension(self, node):
         # TODO: is this how to get the line number for a comprehension?
@@ -595,9 +608,14 @@ class AstArcAnalyzer(object):
 
     handle_AsyncWith = handle_With
 
+    OK_TO_DEFAULT = set([
+        "Assign", "Assert", "AugAssign", "Delete", "Exec", "Expr", "Global",
+        "Import", "ImportFrom", "Pass", "Print",
+    ])
+
     def handle_default(self, node):
         node_name = node.__class__.__name__
-        if node_name not in ["Assign", "Assert", "AugAssign", "Expr", "Import", "Pass", "Print"]:
+        if node_name not in self.OK_TO_DEFAULT:
             # TODO: put 1/0 here to find unhandled nodes.
             print("*** Unhandled: {0}".format(node))
         return set([self.line_for_node(node)])
@@ -610,7 +628,7 @@ class AstArcAnalyzer(object):
         for node in ast.walk(root_node):
             node_name = node.__class__.__name__
             if node_name == "Module":
-                start = self.line_for_node(node.body[0])
+                start = self.line_for_node(node)
                 exits = self.add_body_arcs(node.body, from_line=-1)
                 for exit in exits:
                     self.arcs.add((exit, -start))
-- 
cgit v1.2.1


From 4361522532396635a593a3892dedd8955848d250 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 2 Jan 2016 11:19:45 -0500
Subject: Coding declarations are a pain in the ass

--HG--
branch : ast-branch
---
 coverage/parser.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index fc631fcc..262a78e3 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -17,7 +17,7 @@ from coverage.backward import bytes_to_ints, string_class
 from coverage.bytecode import ByteCodes, CodeObjects
 from coverage.misc import contract, nice_pair, join_regex
 from coverage.misc import CoverageException, NoSource, NotPython
-from coverage.phystokens import compile_unicode, generate_tokens
+from coverage.phystokens import compile_unicode, generate_tokens, neuter_encoding_declaration
 
 
 class PythonParser(object):
@@ -324,8 +324,9 @@ class TryBlock(object):
 
 
 class AstArcAnalyzer(object):
+    @contract(text='unicode')
     def __init__(self, text):
-        self.root_node = ast.parse(text)
+        self.root_node = ast.parse(neuter_encoding_declaration(text))
         if int(os.environ.get("COVERAGE_ASTDUMP", 0)):
             # Dump the AST so that failing tests have helpful output.
             ast_dump(self.root_node)
-- 
cgit v1.2.1


From f98d5bfb6e939f046478b502e2041ac82f91632d Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 2 Jan 2016 14:30:28 -0500
Subject: Better exception support, include except-except arcs, and except-else

--HG--
branch : ast-branch
---
 coverage/parser.py | 81 ++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 60 insertions(+), 21 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 262a78e3..44cb1559 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -540,41 +540,56 @@ class AstArcAnalyzer(object):
         return set()
 
     def handle_Try(self, node):
-        return self.try_work(node, node.body, node.handlers, node.orelse, node.finalbody)
-
-    def handle_TryExcept(self, node):
-        return self.try_work(node, node.body, node.handlers, node.orelse, None)
-
-    def handle_TryFinally(self, node):
-        return self.try_work(node, node.body, None, None, node.finalbody)
-
-    def try_work(self, node, body, handlers, orelse, finalbody):
         # try/finally is tricky. If there's a finally clause, then we need a
         # FinallyBlock to track what flows might go through the finally instead
         # of their normal flow.
-        if handlers:
-            handler_start = self.line_for_node(handlers[0])
+        if node.handlers:
+            handler_start = self.line_for_node(node.handlers[0])
         else:
             handler_start = None
-        if finalbody:
-            final_start = self.line_for_node(finalbody[0])
+
+        if node.finalbody:
+            final_start = self.line_for_node(node.finalbody[0])
         else:
             final_start = None
+
         self.block_stack.append(TryBlock(handler_start=handler_start, final_start=final_start))
+
         start = self.line_for_node(node)
-        exits = self.add_body_arcs(body, from_line=start)
+        exits = self.add_body_arcs(node.body, from_line=start)
+
         try_block = self.block_stack.pop()
         handler_exits = set()
-        if handlers:
-            for handler_node in handlers:
+        last_handler_start = None
+        if node.handlers:
+            for handler_node in node.handlers:
                 handler_start = self.line_for_node(handler_node)
-                # TODO: handler_node.name and handler_node.type
+                if last_handler_start is not None:
+                    self.arcs.add((last_handler_start, handler_start))
+                last_handler_start = handler_start
                 handler_exits |= self.add_body_arcs(handler_node.body, from_line=handler_start)
-        # TODO: node.orelse
+                if handler_node.type is None:
+                    # "except:" doesn't jump to subsequent handlers, or
+                    # "finally:".
+                    last_handler_start = None
+                    # TODO: should we break here? Handlers after "except:"
+                    # won't be run.  Should coverage know that code can't be
+                    # run, or should it flag it as not run?
+
+        if node.orelse:
+            exits = self.add_body_arcs(node.orelse, prev_lines=exits)
+
         exits |= handler_exits
-        if finalbody:
-            final_from = exits | try_block.break_from | try_block.continue_from | try_block.raise_from | try_block.return_from
-            exits = self.add_body_arcs(finalbody, prev_lines=final_from)
+        if node.finalbody:
+            final_from = exits | try_block.break_from | try_block.continue_from | try_block.return_from
+            if node.handlers and last_handler_start is not None:
+                # If there was an "except X:" clause, then a "raise" in the
+                # body goes to the "except X:" before the "finally", but the
+                # "except" go to the finally.
+                final_from.add(last_handler_start)
+            else:
+                final_from |= try_block.raise_from
+            exits = self.add_body_arcs(node.finalbody, prev_lines=final_from)
             if try_block.break_from:
                 self.process_break_exits(exits)
             if try_block.continue_from:
@@ -585,6 +600,30 @@ class AstArcAnalyzer(object):
                 self.process_return_exits(exits)
         return exits
 
+    def handle_TryExcept(self, node):
+        # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
+        # TryExcept, it means there was no finally, so fake it, and treat as
+        # a general Try node.
+        node.finalbody = []
+        return self.handle_Try(node)
+
+    def handle_TryFinally(self, node):
+        # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
+        # TryFinally, see if there's a TryExcept nested inside. If so, merge
+        # them. Otherwise, fake fields to complete a Try node.
+        node.handlers = []
+        node.orelse = []
+
+        if node.body:
+            first = node.body[0]
+            if first.__class__.__name__ == "TryExcept" and node.lineno == first.lineno:
+                assert len(node.body) == 1
+                node.body = first.body
+                node.handlers = first.handlers
+                node.orelse = first.orelse
+
+        return self.handle_Try(node)
+
     def handle_While(self, node):
         constant_test = self.is_constant_expr(node.test)
         start = to_top = self.line_for_node(node.test)
-- 
cgit v1.2.1


From 3440e214df5ddd0f507ecd76c2350eb8d9dd6a75 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 2 Jan 2016 16:14:35 -0500
Subject: Support returning through a finally

--HG--
branch : ast-branch
---
 coverage/parser.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 44cb1559..d85f0b57 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -426,8 +426,6 @@ class AstArcAnalyzer(object):
     # TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
     # TODO: multi-line listcomps
     # TODO: nested function definitions
-    # TODO: multiple `except` clauses
-    # TODO: return->finally
 
     def process_break_exits(self, exits):
         for block in self.blocks():
@@ -467,8 +465,10 @@ class AstArcAnalyzer(object):
 
     def process_return_exits(self, exits):
         for block in self.blocks():
-            # TODO: need a check here for TryBlock
-            if isinstance(block, FunctionBlock):
+            if isinstance(block, TryBlock) and block.final_start:
+                block.return_from.update(exits)
+                break
+            elif isinstance(block, FunctionBlock):
                 # TODO: what if there is no enclosing function?
                 for exit in exits:
                     self.arcs.add((exit, -block.start))
-- 
cgit v1.2.1


From a2cb815e890f092822fa211713ff3d33887afd86 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sun, 3 Jan 2016 11:08:06 -0500
Subject: Fix arcs for function and class decorators

--HG--
branch : ast-branch
---
 coverage/parser.py | 43 +++++++++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 10 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index c11bc222..d3fbad83 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -67,8 +67,9 @@ class PythonParser(object):
         # The raw line numbers of excluded lines of code, as marked by pragmas.
         self.raw_excluded = set()
 
-        # The line numbers of class definitions.
+        # The line numbers of class and function definitions.
         self.raw_classdefs = set()
+        self.raw_funcdefs = set()
 
         # The line numbers of docstring lines.
         self.raw_docstrings = set()
@@ -146,6 +147,8 @@ class PythonParser(object):
                     # we need to exclude them.  The simplest way is to note the
                     # lines with the 'class' keyword.
                     self.raw_classdefs.add(slineno)
+                elif ttext == 'def':
+                    self.raw_funcdefs.add(slineno)
             elif toktype == token.OP:
                 if ttext == ':':
                     should_exclude = (elineno in self.raw_excluded) or excluding_decorators
@@ -268,7 +271,7 @@ class PythonParser(object):
 
     def arcs(self):
         if self._all_arcs is None:
-            aaa = AstArcAnalyzer(self.text)
+            aaa = AstArcAnalyzer(self.text, self.raw_funcdefs, self.raw_classdefs)
             arcs = aaa.collect_arcs()
 
             self._all_arcs = set()
@@ -327,9 +330,12 @@ class TryBlock(object):
 
 
 class AstArcAnalyzer(object):
-    @contract(text='unicode')
-    def __init__(self, text):
+    @contract(text='unicode', funcdefs=set, classdefs=set)
+    def __init__(self, text, funcdefs, classdefs):
         self.root_node = ast.parse(neuter_encoding_declaration(text))
+        self.funcdefs = funcdefs
+        self.classdefs = classdefs
+
         if int(os.environ.get("COVERAGE_ASTDUMP", 0)):
             # Dump the AST so that failing tests have helpful output.
             ast_dump(self.root_node)
@@ -485,9 +491,25 @@ class AstArcAnalyzer(object):
         return set()
 
     def handle_ClassDef(self, node):
-        start = self.line_for_node(node)
+        return self.do_decorated(node, self.classdefs)
+
+    def do_decorated(self, node, defs):
+        first = last = self.line_for_node(node)
+        if node.decorator_list:
+            for dec_node in node.decorator_list:
+                dec_start = self.line_for_node(dec_node)
+                if dec_start != last:
+                    self.arcs.add((last, dec_start))
+                last = dec_start
+        # The definition line may have been missed, but we should have it in
+        # `defs`.
+        body_start = self.line_for_node(node.body[0])
+        for lineno in range(last+1, body_start):
+            if lineno in defs:
+                self.arcs.add((last, lineno))
+                last = lineno
         # the body is handled in add_arcs_for_code_objects.
-        return set([start])
+        return set([last])
 
     def handle_Continue(self, node):
         here = self.line_for_node(node)
@@ -515,9 +537,7 @@ class AstArcAnalyzer(object):
     handle_AsyncFor = handle_For
 
     def handle_FunctionDef(self, node):
-        start = self.line_for_node(node)
-        # the body is handled in add_arcs_for_code_objects.
-        return set([start])
+        return self.do_decorated(node, self.funcdefs)
 
     handle_AsyncFunctionDef = handle_FunctionDef
 
@@ -1159,7 +1179,10 @@ def ast_dump(node, depth=0):
         print("{0} {1}: {2!r}>".format(head, field_name, value))
     else:
         print(head)
-        print("{0}# mro: {1}".format(indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:])))
+        if 0:
+            print("{0}# mro: {1}".format(
+                indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:]),
+            ))
         next_indent = indent + "    "
         for field_name, value in named_fields:
             prefix = "{0}{1}:".format(next_indent, field_name)
-- 
cgit v1.2.1


From 1aa9abd82ecde6d5181a17082f666baca00198ef Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sun, 3 Jan 2016 12:31:58 -0500
Subject: Clean up some lint

--HG--
branch : ast-branch
---
 coverage/parser.py | 56 ++++++++++++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 27 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index d3fbad83..39e23d23 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -399,7 +399,7 @@ class AstArcAnalyzer(object):
         # TODO: this is probably over-doing it, and too expensive. Can we
         # instrument the ast walking to see how many nodes we are revisiting?
         if isinstance(node, ast.stmt):
-            for name, value in ast.iter_fields(node):
+            for _, value in ast.iter_fields(node):
                 if isinstance(value, ast.expr) and self.contains_return_expression(value):
                     self.process_return_exits([self.line_for_node(node)])
                     break
@@ -450,8 +450,8 @@ class AstArcAnalyzer(object):
         for block in self.blocks():
             if isinstance(block, LoopBlock):
                 # TODO: what if there is no loop?
-                for exit in exits:
-                    self.arcs.add((exit, block.start))
+                for xit in exits:
+                    self.arcs.add((xit, block.start))
                 break
             elif isinstance(block, TryBlock) and block.final_start:
                 block.continue_from.update(exits)
@@ -461,15 +461,15 @@ class AstArcAnalyzer(object):
         for block in self.blocks():
             if isinstance(block, TryBlock):
                 if block.handler_start:
-                    for exit in exits:
-                        self.arcs.add((exit, block.handler_start))
+                    for xit in exits:
+                        self.arcs.add((xit, block.handler_start))
                     break
                 elif block.final_start:
                     block.raise_from.update(exits)
                     break
             elif isinstance(block, FunctionBlock):
-                for exit in exits:
-                    self.arcs.add((exit, -block.start))
+                for xit in exits:
+                    self.arcs.add((xit, -block.start))
                 break
 
     def process_return_exits(self, exits):
@@ -479,8 +479,8 @@ class AstArcAnalyzer(object):
                 break
             elif isinstance(block, FunctionBlock):
                 # TODO: what if there is no enclosing function?
-                for exit in exits:
-                    self.arcs.add((exit, -block.start))
+                for xit in exits:
+                    self.arcs.add((xit, -block.start))
                 break
 
     ## Handlers
@@ -491,10 +491,10 @@ class AstArcAnalyzer(object):
         return set()
 
     def handle_ClassDef(self, node):
-        return self.do_decorated(node, self.classdefs)
+        return self.process_decorated(node, self.classdefs)
 
-    def do_decorated(self, node, defs):
-        first = last = self.line_for_node(node)
+    def process_decorated(self, node, defs):
+        last = self.line_for_node(node)
         if node.decorator_list:
             for dec_node in node.decorator_list:
                 dec_start = self.line_for_node(dec_node)
@@ -520,8 +520,8 @@ class AstArcAnalyzer(object):
         start = self.line_for_node(node.iter)
         self.block_stack.append(LoopBlock(start=start))
         exits = self.add_body_arcs(node.body, from_line=start)
-        for exit in exits:
-            self.arcs.add((exit, start))
+        for xit in exits:
+            self.arcs.add((xit, start))
         my_block = self.block_stack.pop()
         exits = my_block.break_exits
         if node.orelse:
@@ -537,7 +537,7 @@ class AstArcAnalyzer(object):
     handle_AsyncFor = handle_For
 
     def handle_FunctionDef(self, node):
-        return self.do_decorated(node, self.funcdefs)
+        return self.process_decorated(node, self.funcdefs)
 
     handle_AsyncFunctionDef = handle_FunctionDef
 
@@ -547,9 +547,6 @@ class AstArcAnalyzer(object):
         exits |= self.add_body_arcs(node.orelse, from_line=start)
         return exits
 
-    def handle_Module(self, node):
-        raise Exception("TODO: this shouldn't happen")
-
     def handle_Raise(self, node):
         # `raise` statement jumps away, no exits from here.
         here = self.line_for_node(node)
@@ -604,7 +601,12 @@ class AstArcAnalyzer(object):
 
         exits |= handler_exits
         if node.finalbody:
-            final_from = exits | try_block.break_from | try_block.continue_from | try_block.return_from
+            final_from = (                  # You can get to the `finally` clause from:
+                exits |                         # the exits of the body or `else` clause,
+                try_block.break_from |          # or a `break` in the body,
+                try_block.continue_from |       # or a `continue` in the body,
+                try_block.return_from           # or a `return` in the body.
+            )
             if node.handlers and last_handler_start is not None:
                 # If there was an "except X:" clause, then a "raise" in the
                 # body goes to the "except X:" before the "finally", but the
@@ -654,8 +656,8 @@ class AstArcAnalyzer(object):
             to_top = self.line_for_node(node.body[0])
         self.block_stack.append(LoopBlock(start=start))
         exits = self.add_body_arcs(node.body, from_line=start)
-        for exit in exits:
-            self.arcs.add((exit, to_top))
+        for xit in exits:
+            self.arcs.add((xit, to_top))
         exits = set()
         if not constant_test:
             exits.add(start)
@@ -693,21 +695,21 @@ class AstArcAnalyzer(object):
             if node_name == "Module":
                 start = self.line_for_node(node)
                 exits = self.add_body_arcs(node.body, from_line=-1)
-                for exit in exits:
-                    self.arcs.add((exit, -start))
+                for xit in exits:
+                    self.arcs.add((xit, -start))
             elif node_name in ["FunctionDef", "AsyncFunctionDef"]:
                 start = self.line_for_node(node)
                 self.block_stack.append(FunctionBlock(start=start))
                 exits = self.add_body_arcs(node.body, from_line=-1)
                 self.block_stack.pop()
-                for exit in exits:
-                    self.arcs.add((exit, -start))
+                for xit in exits:
+                    self.arcs.add((xit, -start))
             elif node_name == "ClassDef":
                 start = self.line_for_node(node)
                 self.arcs.add((-1, start))
                 exits = self.add_body_arcs(node.body, from_line=start)
-                for exit in exits:
-                    self.arcs.add((exit, -start))
+                for xit in exits:
+                    self.arcs.add((xit, -start))
             elif node_name in self.CODE_COMPREHENSIONS:
                 # TODO: tests for when generators is more than one?
                 for gen in node.generators:
-- 
cgit v1.2.1


From 4074315ac65ed79e94bc331a8059859781b5b12b Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Mon, 4 Jan 2016 20:12:55 -0500
Subject: Support comprehensions better

--HG--
branch : ast-branch
---
 coverage/parser.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 39e23d23..c680f63b 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -383,10 +383,6 @@ class AstArcAnalyzer(object):
             # TODO: test case for empty module.
             return 1
 
-    def line_comprehension(self, node):
-        # TODO: is this how to get the line number for a comprehension?
-        return node.target.lineno
-
     def line_default(self, node):
         return node.lineno
 
@@ -433,7 +429,6 @@ class AstArcAnalyzer(object):
     # TODO: multi-target assignment with computed targets
     # TODO: listcomps hidden deep in other expressions
     # TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
-    # TODO: multi-line listcomps
     # TODO: nested function definitions
 
     def process_break_exits(self, exits):
@@ -554,7 +549,6 @@ class AstArcAnalyzer(object):
         return set()
 
     def handle_Return(self, node):
-        # TODO: deal with returning through a finally.
         here = self.line_for_node(node)
         self.process_return_exits([here])
         return set()
@@ -711,12 +705,9 @@ class AstArcAnalyzer(object):
                 for xit in exits:
                     self.arcs.add((xit, -start))
             elif node_name in self.CODE_COMPREHENSIONS:
-                # TODO: tests for when generators is more than one?
-                for gen in node.generators:
-                    start = self.line_for_node(gen)
-                    self.arcs.add((-1, start))
-                    self.arcs.add((start, -start))
-                    # TODO: guaranteed this won't work for multi-line comps.
+                start = self.line_for_node(node)
+                self.arcs.add((-1, start))
+                self.arcs.add((start, -start))
             elif node_name == "Lambda":
                 start = self.line_for_node(node)
                 self.arcs.add((-1, start))
-- 
cgit v1.2.1


From f7f56ec9adaa531019a27ef7c634db816f30040a Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 5 Jan 2016 06:54:07 -0500
Subject: Support while-else

--HG--
branch : ast-branch
---
 coverage/parser.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index c680f63b..b0e7371f 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -520,8 +520,6 @@ class AstArcAnalyzer(object):
         my_block = self.block_stack.pop()
         exits = my_block.break_exits
         if node.orelse:
-            else_start = self.line_for_node(node.orelse[0])
-            self.arcs.add((start, else_start))
             else_exits = self.add_body_arcs(node.orelse, from_line=start)
             exits |= else_exits
         else:
@@ -653,11 +651,15 @@ class AstArcAnalyzer(object):
         for xit in exits:
             self.arcs.add((xit, to_top))
         exits = set()
-        if not constant_test:
-            exits.add(start)
         my_block = self.block_stack.pop()
         exits.update(my_block.break_exits)
-        # TODO: orelse
+        if node.orelse:
+            else_exits = self.add_body_arcs(node.orelse, from_line=start)
+            exits |= else_exits
+        else:
+            # No `else` clause: you can exit from the start.
+            if not constant_test:
+                exits.add(start)
         return exits
 
     def handle_With(self, node):
-- 
cgit v1.2.1


From 8f9b4f9d596ef4a5c0d26b4e54acfcd0558ece39 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Wed, 6 Jan 2016 07:11:32 -0500
Subject: Add some tests for uncovered cases

--HG--
branch : ast-branch
---
 coverage/parser.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index b0e7371f..a6a8ad65 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -336,7 +336,7 @@ class AstArcAnalyzer(object):
         self.funcdefs = funcdefs
         self.classdefs = classdefs
 
-        if int(os.environ.get("COVERAGE_ASTDUMP", 0)):
+        if int(os.environ.get("COVERAGE_ASTDUMP", 0)):      # pragma: debugging
             # Dump the AST so that failing tests have helpful output.
             ast_dump(self.root_node)
 
@@ -372,7 +372,6 @@ class AstArcAnalyzer(object):
         if node.elts:
             return self.line_for_node(node.elts[0])
         else:
-            # TODO: test case for this branch:  x = []
             return node.lineno
 
     def line_Module(self, node):
@@ -380,7 +379,6 @@ class AstArcAnalyzer(object):
             return self.line_for_node(node.body[0])
         else:
             # Modules have no line number, they always start at 1.
-            # TODO: test case for empty module.
             return 1
 
     def line_default(self, node):
@@ -426,7 +424,6 @@ class AstArcAnalyzer(object):
     # tests to write:
     # TODO: while EXPR:
     # TODO: while False:
-    # TODO: multi-target assignment with computed targets
     # TODO: listcomps hidden deep in other expressions
     # TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
     # TODO: nested function definitions
@@ -688,11 +685,17 @@ class AstArcAnalyzer(object):
     def add_arcs_for_code_objects(self, root_node):
         for node in ast.walk(root_node):
             node_name = node.__class__.__name__
+            # TODO: should this be broken into separate methods?
             if node_name == "Module":
                 start = self.line_for_node(node)
-                exits = self.add_body_arcs(node.body, from_line=-1)
-                for xit in exits:
-                    self.arcs.add((xit, -start))
+                if node.body:
+                    exits = self.add_body_arcs(node.body, from_line=-1)
+                    for xit in exits:
+                        self.arcs.add((xit, -start))
+                else:
+                    # Empty module.
+                    self.arcs.add((-1, start))
+                    self.arcs.add((start, -1))
             elif node_name in ["FunctionDef", "AsyncFunctionDef"]:
                 start = self.line_for_node(node)
                 self.block_stack.append(FunctionBlock(start=start))
-- 
cgit v1.2.1


From 909e0afcc474d7ede12e2f967bbc34097e132915 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Wed, 6 Jan 2016 07:56:05 -0500
Subject: Clean up some TODO's and code paths

--HG--
branch : ast-branch
---
 coverage/parser.py | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index a6a8ad65..348eb7c5 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -431,7 +431,6 @@ class AstArcAnalyzer(object):
     def process_break_exits(self, exits):
         for block in self.blocks():
             if isinstance(block, LoopBlock):
-                # TODO: what if there is no loop?
                 block.break_exits.update(exits)
                 break
             elif isinstance(block, TryBlock) and block.final_start:
@@ -441,7 +440,6 @@ class AstArcAnalyzer(object):
     def process_continue_exits(self, exits):
         for block in self.blocks():
             if isinstance(block, LoopBlock):
-                # TODO: what if there is no loop?
                 for xit in exits:
                     self.arcs.add((xit, block.start))
                 break
@@ -470,7 +468,6 @@ class AstArcAnalyzer(object):
                 block.return_from.update(exits)
                 break
             elif isinstance(block, FunctionBlock):
-                # TODO: what if there is no enclosing function?
                 for xit in exits:
                     self.arcs.add((xit, -block.start))
                 break
@@ -628,13 +625,12 @@ class AstArcAnalyzer(object):
         node.handlers = []
         node.orelse = []
 
-        if node.body:
-            first = node.body[0]
-            if first.__class__.__name__ == "TryExcept" and node.lineno == first.lineno:
-                assert len(node.body) == 1
-                node.body = first.body
-                node.handlers = first.handlers
-                node.orelse = first.orelse
+        first = node.body[0]
+        if first.__class__.__name__ == "TryExcept" and node.lineno == first.lineno:
+            assert len(node.body) == 1
+            node.body = first.body
+            node.handlers = first.handlers
+            node.orelse = first.orelse
 
         return self.handle_Try(node)
 
@@ -672,10 +668,10 @@ class AstArcAnalyzer(object):
     ])
 
     def handle_default(self, node):
-        node_name = node.__class__.__name__
-        if node_name not in self.OK_TO_DEFAULT:
-            # TODO: put 1/0 here to find unhandled nodes.
-            print("*** Unhandled: {0}".format(node))
+        if 0:
+            node_name = node.__class__.__name__
+            if node_name not in self.OK_TO_DEFAULT:
+                print("*** Unhandled: {0}".format(node))
         return set([self.line_for_node(node)])
 
     CODE_COMPREHENSIONS = set(["GeneratorExp", "DictComp", "SetComp"])
-- 
cgit v1.2.1


From 426961555ee866da2addb74b25c6dfca3d2c5f33 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Wed, 6 Jan 2016 08:11:58 -0500
Subject: More uniform dispatch: use methods for everything, and handle
 defaults in the dispatch instead of calling another method.

--HG--
branch : ast-branch
---
 coverage/parser.py | 120 +++++++++++++++++++++++++++++------------------------
 1 file changed, 66 insertions(+), 54 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 348eb7c5..c5d7c618 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -355,8 +355,11 @@ class AstArcAnalyzer(object):
     def line_for_node(self, node):
         """What is the right line number to use for this node?"""
         node_name = node.__class__.__name__
-        handler = getattr(self, "line_" + node_name, self.line_default)
-        return handler(node)
+        handler = getattr(self, "line_" + node_name, None)
+        if handler is not None:
+            return handler(node)
+        else:
+            return node.lineno
 
     def line_Assign(self, node):
         return self.line_for_node(node.value)
@@ -381,8 +384,10 @@ class AstArcAnalyzer(object):
             # Modules have no line number, they always start at 1.
             return 1
 
-    def line_default(self, node):
-        return node.lineno
+    OK_TO_DEFAULT = set([
+        "Assign", "Assert", "AugAssign", "Delete", "Exec", "Expr", "Global",
+        "Import", "ImportFrom", "Pass", "Print",
+    ])
 
     def add_arcs(self, node):
         """Add the arcs for `node`.
@@ -397,9 +402,17 @@ class AstArcAnalyzer(object):
                 if isinstance(value, ast.expr) and self.contains_return_expression(value):
                     self.process_return_exits([self.line_for_node(node)])
                     break
+
         node_name = node.__class__.__name__
-        handler = getattr(self, "handle_" + node_name, self.handle_default)
-        return handler(node)
+        handler = getattr(self, "handle_" + node_name, None)
+        if handler is not None:
+            return handler(node)
+
+        if 0:
+            node_name = node.__class__.__name__
+            if node_name not in self.OK_TO_DEFAULT:
+                print("*** Unhandled: {0}".format(node))
+        return set([self.line_for_node(node)])
 
     def add_body_arcs(self, body, from_line=None, prev_lines=None):
         if prev_lines is None:
@@ -662,58 +675,57 @@ class AstArcAnalyzer(object):
 
     handle_AsyncWith = handle_With
 
-    OK_TO_DEFAULT = set([
-        "Assign", "Assert", "AugAssign", "Delete", "Exec", "Expr", "Global",
-        "Import", "ImportFrom", "Pass", "Print",
-    ])
-
-    def handle_default(self, node):
-        if 0:
+    def add_arcs_for_code_objects(self, root_node):
+        for node in ast.walk(root_node):
             node_name = node.__class__.__name__
-            if node_name not in self.OK_TO_DEFAULT:
-                print("*** Unhandled: {0}".format(node))
-        return set([self.line_for_node(node)])
+            code_object_handler = getattr(self, "code_object_" + node_name, None)
+            if code_object_handler is not None:
+                code_object_handler(node)
+
+    def code_object_Module(self, node):
+        start = self.line_for_node(node)
+        if node.body:
+            exits = self.add_body_arcs(node.body, from_line=-1)
+            for xit in exits:
+                self.arcs.add((xit, -start))
+        else:
+            # Empty module.
+            self.arcs.add((-1, start))
+            self.arcs.add((start, -1))
+
+    def code_object_FunctionDef(self, node):
+        start = self.line_for_node(node)
+        self.block_stack.append(FunctionBlock(start=start))
+        exits = self.add_body_arcs(node.body, from_line=-1)
+        self.block_stack.pop()
+        for xit in exits:
+            self.arcs.add((xit, -start))
+
+    code_object_AsyncFunctionDef = code_object_FunctionDef
 
-    CODE_COMPREHENSIONS = set(["GeneratorExp", "DictComp", "SetComp"])
+    def code_object_ClassDef(self, node):
+        start = self.line_for_node(node)
+        self.arcs.add((-1, start))
+        exits = self.add_body_arcs(node.body, from_line=start)
+        for xit in exits:
+            self.arcs.add((xit, -start))
+
+    def do_code_object_comprehension(self, node):
+        start = self.line_for_node(node)
+        self.arcs.add((-1, start))
+        self.arcs.add((start, -start))
+
+    code_object_GeneratorExp = do_code_object_comprehension
+    code_object_DictComp = do_code_object_comprehension
+    code_object_SetComp = do_code_object_comprehension
     if env.PY3:
-        CODE_COMPREHENSIONS.add("ListComp")
+        code_object_ListComp = do_code_object_comprehension
 
-    def add_arcs_for_code_objects(self, root_node):
-        for node in ast.walk(root_node):
-            node_name = node.__class__.__name__
-            # TODO: should this be broken into separate methods?
-            if node_name == "Module":
-                start = self.line_for_node(node)
-                if node.body:
-                    exits = self.add_body_arcs(node.body, from_line=-1)
-                    for xit in exits:
-                        self.arcs.add((xit, -start))
-                else:
-                    # Empty module.
-                    self.arcs.add((-1, start))
-                    self.arcs.add((start, -1))
-            elif node_name in ["FunctionDef", "AsyncFunctionDef"]:
-                start = self.line_for_node(node)
-                self.block_stack.append(FunctionBlock(start=start))
-                exits = self.add_body_arcs(node.body, from_line=-1)
-                self.block_stack.pop()
-                for xit in exits:
-                    self.arcs.add((xit, -start))
-            elif node_name == "ClassDef":
-                start = self.line_for_node(node)
-                self.arcs.add((-1, start))
-                exits = self.add_body_arcs(node.body, from_line=start)
-                for xit in exits:
-                    self.arcs.add((xit, -start))
-            elif node_name in self.CODE_COMPREHENSIONS:
-                start = self.line_for_node(node)
-                self.arcs.add((-1, start))
-                self.arcs.add((start, -start))
-            elif node_name == "Lambda":
-                start = self.line_for_node(node)
-                self.arcs.add((-1, start))
-                self.arcs.add((start, -start))
-                # TODO: test multi-line lambdas
+    def code_object_Lambda(self, node):
+        start = self.line_for_node(node)
+        self.arcs.add((-1, start))
+        self.arcs.add((start, -start))
+        # TODO: test multi-line lambdas
 
     def contains_return_expression(self, node):
         """Is there a yield-from or await in `node` someplace?"""
-- 
cgit v1.2.1


From badf53c6cb26118cfdf3388a6eb09fd21e6c7428 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Wed, 6 Jan 2016 08:46:59 -0500
Subject: Name the dispatched-to methods more unusually

--HG--
branch : ast-branch
---
 coverage/parser.py | 68 +++++++++++++++++++++++++++---------------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index c5d7c618..647dbd05 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -355,29 +355,29 @@ class AstArcAnalyzer(object):
     def line_for_node(self, node):
         """What is the right line number to use for this node?"""
         node_name = node.__class__.__name__
-        handler = getattr(self, "line_" + node_name, None)
+        handler = getattr(self, "_line__" + node_name, None)
         if handler is not None:
             return handler(node)
         else:
             return node.lineno
 
-    def line_Assign(self, node):
+    def _line__Assign(self, node):
         return self.line_for_node(node.value)
 
-    def line_Dict(self, node):
+    def _line__Dict(self, node):
         # Python 3.5 changed how dict literals are made.
         if env.PYVERSION >= (3, 5) and node.keys:
             return node.keys[0].lineno
         else:
             return node.lineno
 
-    def line_List(self, node):
+    def _line__List(self, node):
         if node.elts:
             return self.line_for_node(node.elts[0])
         else:
             return node.lineno
 
-    def line_Module(self, node):
+    def _line__Module(self, node):
         if node.body:
             return self.line_for_node(node.body[0])
         else:
@@ -404,7 +404,7 @@ class AstArcAnalyzer(object):
                     break
 
         node_name = node.__class__.__name__
-        handler = getattr(self, "handle_" + node_name, None)
+        handler = getattr(self, "_handle__" + node_name, None)
         if handler is not None:
             return handler(node)
 
@@ -487,12 +487,12 @@ class AstArcAnalyzer(object):
 
     ## Handlers
 
-    def handle_Break(self, node):
+    def _handle__Break(self, node):
         here = self.line_for_node(node)
         self.process_break_exits([here])
         return set()
 
-    def handle_ClassDef(self, node):
+    def _handle__ClassDef(self, node):
         return self.process_decorated(node, self.classdefs)
 
     def process_decorated(self, node, defs):
@@ -513,12 +513,12 @@ class AstArcAnalyzer(object):
         # the body is handled in add_arcs_for_code_objects.
         return set([last])
 
-    def handle_Continue(self, node):
+    def _handle__Continue(self, node):
         here = self.line_for_node(node)
         self.process_continue_exits([here])
         return set()
 
-    def handle_For(self, node):
+    def _handle__For(self, node):
         start = self.line_for_node(node.iter)
         self.block_stack.append(LoopBlock(start=start))
         exits = self.add_body_arcs(node.body, from_line=start)
@@ -534,31 +534,31 @@ class AstArcAnalyzer(object):
             exits.add(start)
         return exits
 
-    handle_AsyncFor = handle_For
+    _handle__AsyncFor = _handle__For
 
-    def handle_FunctionDef(self, node):
+    def _handle__FunctionDef(self, node):
         return self.process_decorated(node, self.funcdefs)
 
-    handle_AsyncFunctionDef = handle_FunctionDef
+    _handle__AsyncFunctionDef = _handle__FunctionDef
 
-    def handle_If(self, node):
+    def _handle__If(self, node):
         start = self.line_for_node(node.test)
         exits = self.add_body_arcs(node.body, from_line=start)
         exits |= self.add_body_arcs(node.orelse, from_line=start)
         return exits
 
-    def handle_Raise(self, node):
+    def _handle__Raise(self, node):
         # `raise` statement jumps away, no exits from here.
         here = self.line_for_node(node)
         self.process_raise_exits([here])
         return set()
 
-    def handle_Return(self, node):
+    def _handle__Return(self, node):
         here = self.line_for_node(node)
         self.process_return_exits([here])
         return set()
 
-    def handle_Try(self, node):
+    def _handle__Try(self, node):
         # try/finally is tricky. If there's a finally clause, then we need a
         # FinallyBlock to track what flows might go through the finally instead
         # of their normal flow.
@@ -624,14 +624,14 @@ class AstArcAnalyzer(object):
                 self.process_return_exits(exits)
         return exits
 
-    def handle_TryExcept(self, node):
+    def _handle__TryExcept(self, node):
         # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
         # TryExcept, it means there was no finally, so fake it, and treat as
         # a general Try node.
         node.finalbody = []
-        return self.handle_Try(node)
+        return self._handle__Try(node)
 
-    def handle_TryFinally(self, node):
+    def _handle__TryFinally(self, node):
         # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
         # TryFinally, see if there's a TryExcept nested inside. If so, merge
         # them. Otherwise, fake fields to complete a Try node.
@@ -645,9 +645,9 @@ class AstArcAnalyzer(object):
             node.handlers = first.handlers
             node.orelse = first.orelse
 
-        return self.handle_Try(node)
+        return self._handle__Try(node)
 
-    def handle_While(self, node):
+    def _handle__While(self, node):
         constant_test = self.is_constant_expr(node.test)
         start = to_top = self.line_for_node(node.test)
         if constant_test:
@@ -668,21 +668,21 @@ class AstArcAnalyzer(object):
                 exits.add(start)
         return exits
 
-    def handle_With(self, node):
+    def _handle__With(self, node):
         start = self.line_for_node(node)
         exits = self.add_body_arcs(node.body, from_line=start)
         return exits
 
-    handle_AsyncWith = handle_With
+    _handle__AsyncWith = _handle__With
 
     def add_arcs_for_code_objects(self, root_node):
         for node in ast.walk(root_node):
             node_name = node.__class__.__name__
-            code_object_handler = getattr(self, "code_object_" + node_name, None)
+            code_object_handler = getattr(self, "_code_object__" + node_name, None)
             if code_object_handler is not None:
                 code_object_handler(node)
 
-    def code_object_Module(self, node):
+    def _code_object__Module(self, node):
         start = self.line_for_node(node)
         if node.body:
             exits = self.add_body_arcs(node.body, from_line=-1)
@@ -693,7 +693,7 @@ class AstArcAnalyzer(object):
             self.arcs.add((-1, start))
             self.arcs.add((start, -1))
 
-    def code_object_FunctionDef(self, node):
+    def _code_object__FunctionDef(self, node):
         start = self.line_for_node(node)
         self.block_stack.append(FunctionBlock(start=start))
         exits = self.add_body_arcs(node.body, from_line=-1)
@@ -701,9 +701,9 @@ class AstArcAnalyzer(object):
         for xit in exits:
             self.arcs.add((xit, -start))
 
-    code_object_AsyncFunctionDef = code_object_FunctionDef
+    _code_object__AsyncFunctionDef = _code_object__FunctionDef
 
-    def code_object_ClassDef(self, node):
+    def _code_object__ClassDef(self, node):
         start = self.line_for_node(node)
         self.arcs.add((-1, start))
         exits = self.add_body_arcs(node.body, from_line=start)
@@ -715,13 +715,13 @@ class AstArcAnalyzer(object):
         self.arcs.add((-1, start))
         self.arcs.add((start, -start))
 
-    code_object_GeneratorExp = do_code_object_comprehension
-    code_object_DictComp = do_code_object_comprehension
-    code_object_SetComp = do_code_object_comprehension
+    _code_object__GeneratorExp = do_code_object_comprehension
+    _code_object__DictComp = do_code_object_comprehension
+    _code_object__SetComp = do_code_object_comprehension
     if env.PY3:
-        code_object_ListComp = do_code_object_comprehension
+        _code_object__ListComp = do_code_object_comprehension
 
-    def code_object_Lambda(self, node):
+    def _code_object__Lambda(self, node):
         start = self.line_for_node(node)
         self.arcs.add((-1, start))
         self.arcs.add((start, -start))
-- 
cgit v1.2.1


From 8b7c4c1bf2bd0ea40c6da1c9d09f4f978835fa3b Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Wed, 6 Jan 2016 16:27:24 -0500
Subject: Remove the old bytecode-based branch analyzer

--HG--
branch : ast-branch
---
 coverage/parser.py | 361 +----------------------------------------------------
 1 file changed, 2 insertions(+), 359 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 647dbd05..32a75900 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -5,7 +5,6 @@
 
 import ast
 import collections
-import dis
 import os
 import re
 import token
@@ -14,7 +13,7 @@ import tokenize
 from coverage import env
 from coverage.backward import range    # pylint: disable=redefined-builtin
 from coverage.backward import bytes_to_ints, string_class
-from coverage.bytecode import ByteCodes, CodeObjects
+from coverage.bytecode import CodeObjects
 from coverage.misc import contract, nice_pair, join_regex
 from coverage.misc import CoverageException, NoSource, NotPython
 from coverage.phystokens import compile_unicode, generate_tokens, neuter_encoding_declaration
@@ -253,22 +252,6 @@ class PythonParser(object):
         starts = self.raw_statements - ignore
         self.statements = self.first_lines(starts) - ignore
 
-    def old_arcs(self):
-        """Get information about the arcs available in the code.
-
-        Returns a set of line number pairs.  Line numbers have been normalized
-        to the first line of multi-line statements.
-
-        """
-        if self._all_arcs is None:
-            self._all_arcs = set()
-            for l1, l2 in self.byte_parser._all_arcs():
-                fl1 = self.first_line(l1)
-                fl2 = self.first_line(l2)
-                if fl1 != fl2:
-                    self._all_arcs.add((fl1, fl2))
-        return self._all_arcs
-
     def arcs(self):
         if self._all_arcs is None:
             aaa = AstArcAnalyzer(self.text, self.raw_funcdefs, self.raw_classdefs)
@@ -736,62 +719,6 @@ class AstArcAnalyzer(object):
         return False
 
 
-## Opcodes that guide the ByteParser.
-
-def _opcode(name):
-    """Return the opcode by name from the dis module."""
-    return dis.opmap[name]
-
-
-def _opcode_set(*names):
-    """Return a set of opcodes by the names in `names`."""
-    s = set()
-    for name in names:
-        try:
-            s.add(_opcode(name))
-        except KeyError:
-            pass
-    return s
-
-# Opcodes that leave the code object.
-OPS_CODE_END = _opcode_set('RETURN_VALUE')
-
-# Opcodes that unconditionally end the code chunk.
-OPS_CHUNK_END = _opcode_set(
-    'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'RETURN_VALUE', 'RAISE_VARARGS',
-    'BREAK_LOOP', 'CONTINUE_LOOP',
-)
-
-# Opcodes that unconditionally begin a new code chunk.  By starting new chunks
-# with unconditional jump instructions, we neatly deal with jumps to jumps
-# properly.
-OPS_CHUNK_BEGIN = _opcode_set('JUMP_ABSOLUTE', 'JUMP_FORWARD')
-
-# Opcodes that push a block on the block stack.
-OPS_PUSH_BLOCK = _opcode_set(
-    'SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH',
-)
-
-# Block types for exception handling.
-OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY')
-
-# Opcodes that pop a block from the block stack.
-OPS_POP_BLOCK = _opcode_set('POP_BLOCK')
-
-OPS_GET_AITER = _opcode_set('GET_AITER')
-
-# Opcodes that have a jump destination, but aren't really a jump.
-OPS_NO_JUMP = OPS_PUSH_BLOCK
-
-# Individual opcodes we need below.
-OP_BREAK_LOOP = _opcode('BREAK_LOOP')
-OP_END_FINALLY = _opcode('END_FINALLY')
-OP_COMPARE_OP = _opcode('COMPARE_OP')
-COMPARE_EXCEPTION = 10  # just have to get this constant from the code.
-OP_LOAD_CONST = _opcode('LOAD_CONST')
-OP_RETURN_VALUE = _opcode('RETURN_VALUE')
-
-
 class ByteParser(object):
     """Parse byte codes to understand the structure of code."""
 
@@ -812,7 +739,7 @@ class ByteParser(object):
 
         # Alternative Python implementations don't always provide all the
         # attributes on code objects that we need to do the analysis.
-        for attr in ['co_lnotab', 'co_firstlineno', 'co_consts', 'co_code']:
+        for attr in ['co_lnotab', 'co_firstlineno', 'co_consts']:
             if not hasattr(self.code, attr):
                 raise CoverageException(
                     "This implementation of Python doesn't support code analysis.\n"
@@ -867,290 +794,6 @@ class ByteParser(object):
             for _, l in bp._bytes_lines():
                 yield l
 
-    def _block_stack_repr(self, block_stack):               # pragma: debugging
-        """Get a string version of `block_stack`, for debugging."""
-        blocks = ", ".join(
-            "(%s, %r)" % (dis.opname[b[0]], b[1]) for b in block_stack
-        )
-        return "[" + blocks + "]"
-
-    def _split_into_chunks(self):
-        """Split the code object into a list of `Chunk` objects.
-
-        Each chunk is only entered at its first instruction, though there can
-        be many exits from a chunk.
-
-        Returns a list of `Chunk` objects.
-
-        """
-        # The list of chunks so far, and the one we're working on.
-        chunks = []
-        chunk = None
-
-        # A dict mapping byte offsets of line starts to the line numbers.
-        bytes_lines_map = dict(self._bytes_lines())
-
-        # The block stack: loops and try blocks get pushed here for the
-        # implicit jumps that can occur.
-        # Each entry is a tuple: (block type, destination)
-        block_stack = []
-
-        # Some op codes are followed by branches that should be ignored.  This
-        # is a count of how many ignores are left.
-        ignore_branch = 0
-
-        ignore_pop_block = 0
-
-        # We have to handle the last two bytecodes specially.
-        ult = penult = None
-
-        # Get a set of all of the jump-to points.
-        jump_to = set()
-        bytecodes = list(ByteCodes(self.code.co_code))
-        for bc in bytecodes:
-            if bc.jump_to >= 0:
-                jump_to.add(bc.jump_to)
-
-        chunk_lineno = 0
-
-        # Walk the byte codes building chunks.
-        for bc in bytecodes:
-            # Maybe have to start a new chunk.
-            start_new_chunk = False
-            first_chunk = False
-            if bc.offset in bytes_lines_map:
-                # Start a new chunk for each source line number.
-                start_new_chunk = True
-                chunk_lineno = bytes_lines_map[bc.offset]
-                first_chunk = True
-            elif bc.offset in jump_to:
-                # To make chunks have a single entrance, we have to make a new
-                # chunk when we get to a place some bytecode jumps to.
-                start_new_chunk = True
-            elif bc.op in OPS_CHUNK_BEGIN:
-                # Jumps deserve their own unnumbered chunk.  This fixes
-                # problems with jumps to jumps getting confused.
-                start_new_chunk = True
-
-            if not chunk or start_new_chunk:
-                if chunk:
-                    chunk.exits.add(bc.offset)
-                chunk = Chunk(bc.offset, chunk_lineno, first_chunk)
-                if not chunks:
-                    # The very first chunk of a code object is always an
-                    # entrance.
-                    chunk.entrance = True
-                chunks.append(chunk)
-
-            # Look at the opcode.
-            if bc.jump_to >= 0 and bc.op not in OPS_NO_JUMP:
-                if ignore_branch:
-                    # Someone earlier wanted us to ignore this branch.
-                    ignore_branch -= 1
-                else:
-                    # The opcode has a jump, it's an exit for this chunk.
-                    chunk.exits.add(bc.jump_to)
-
-            if bc.op in OPS_CODE_END:
-                # The opcode can exit the code object.
-                chunk.exits.add(-self.code.co_firstlineno)
-            if bc.op in OPS_PUSH_BLOCK:
-                # The opcode adds a block to the block_stack.
-                block_stack.append((bc.op, bc.jump_to))
-            if bc.op in OPS_POP_BLOCK:
-                # The opcode pops a block from the block stack.
-                if ignore_pop_block:
-                    ignore_pop_block -= 1
-                else:
-                    block_stack.pop()
-            if bc.op in OPS_CHUNK_END:
-                # This opcode forces the end of the chunk.
-                if bc.op == OP_BREAK_LOOP:
-                    # A break is implicit: jump where the top of the
-                    # block_stack points.
-                    chunk.exits.add(block_stack[-1][1])
-                chunk = None
-            if bc.op == OP_END_FINALLY:
-                # For the finally clause we need to find the closest exception
-                # block, and use its jump target as an exit.
-                for block in reversed(block_stack):
-                    if block[0] in OPS_EXCEPT_BLOCKS:
-                        chunk.exits.add(block[1])
-                        break
-            if bc.op == OP_COMPARE_OP and bc.arg == COMPARE_EXCEPTION:
-                # This is an except clause.  We want to overlook the next
-                # branch, so that except's don't count as branches.
-                ignore_branch += 1
-
-            if bc.op in OPS_GET_AITER:
-                # GET_AITER is weird: First, it seems to generate one more
-                # POP_BLOCK than SETUP_*, so we have to prepare to ignore one
-                # of the POP_BLOCKS.  Second, we don't have a clear branch to
-                # the exit of the loop, so we peek into the block stack to find
-                # it.
-                ignore_pop_block += 1
-                chunk.exits.add(block_stack[-1][1])
-
-            penult = ult
-            ult = bc
-
-        if chunks:
-            # The last two bytecodes could be a dummy "return None" that
-            # shouldn't be counted as real code. Every Python code object seems
-            # to end with a return, and a "return None" is inserted if there
-            # isn't an explicit return in the source.
-            if ult and penult:
-                if penult.op == OP_LOAD_CONST and ult.op == OP_RETURN_VALUE:
-                    if self.code.co_consts[penult.arg] is None:
-                        # This is "return None", but is it dummy?  A real line
-                        # would be a last chunk all by itself.
-                        if chunks[-1].byte != penult.offset:
-                            ex = -self.code.co_firstlineno
-                            # Split the last chunk
-                            last_chunk = chunks[-1]
-                            last_chunk.exits.remove(ex)
-                            last_chunk.exits.add(penult.offset)
-                            chunk = Chunk(
-                                penult.offset, last_chunk.line, False
-                            )
-                            chunk.exits.add(ex)
-                            chunks.append(chunk)
-
-            # Give all the chunks a length.
-            chunks[-1].length = bc.next_offset - chunks[-1].byte
-            for i in range(len(chunks)-1):
-                chunks[i].length = chunks[i+1].byte - chunks[i].byte
-
-        #self.validate_chunks(chunks)
-        return chunks
-
-    def validate_chunks(self, chunks):                      # pragma: debugging
-        """Validate the rule that chunks have a single entrance."""
-        # starts is the entrances to the chunks
-        starts = set(ch.byte for ch in chunks)
-        for ch in chunks:
-            assert all((ex in starts or ex < 0) for ex in ch.exits)
-
-    def _arcs(self):
-        """Find the executable arcs in the code.
-
-        Yields pairs: (from,to).  From and to are integer line numbers.  If
-        from is < 0, then the arc is an entrance into the code object.  If to
-        is < 0, the arc is an exit from the code object.
-
-        """
-        chunks = self._split_into_chunks()
-
-        # A map from byte offsets to the chunk starting at that offset.
-        byte_chunks = dict((c.byte, c) for c in chunks)
-
-        # Traverse from the first chunk in each line, and yield arcs where
-        # the trace function will be invoked.
-        for chunk in chunks:
-            if chunk.entrance:
-                yield (-1, chunk.line)
-
-            if not chunk.first:
-                continue
-
-            chunks_considered = set()
-            chunks_to_consider = [chunk]
-            while chunks_to_consider:
-                # Get the chunk we're considering, and make sure we don't
-                # consider it again.
-                this_chunk = chunks_to_consider.pop()
-                chunks_considered.add(this_chunk)
-
-                # For each exit, add the line number if the trace function
-                # would be triggered, or add the chunk to those being
-                # considered if not.
-                for ex in this_chunk.exits:
-                    if ex < 0:
-                        yield (chunk.line, ex)
-                    else:
-                        next_chunk = byte_chunks[ex]
-                        if next_chunk in chunks_considered:
-                            continue
-
-                        # The trace function is invoked if visiting the first
-                        # bytecode in a line, or if the transition is a
-                        # backward jump.
-                        backward_jump = next_chunk.byte < this_chunk.byte
-                        if next_chunk.first or backward_jump:
-                            if next_chunk.line != chunk.line:
-                                yield (chunk.line, next_chunk.line)
-                        else:
-                            chunks_to_consider.append(next_chunk)
-
-    def _all_chunks(self):
-        """Returns a list of `Chunk` objects for this code and its children.
-
-        See `_split_into_chunks` for details.
-
-        """
-        chunks = []
-        for bp in self.child_parsers():
-            chunks.extend(bp._split_into_chunks())
-
-        return chunks
-
-    def _all_arcs(self):
-        """Get the set of all arcs in this code object and its children.
-
-        See `_arcs` for details.
-
-        """
-        arcs = set()
-        for bp in self.child_parsers():
-            arcs.update(bp._arcs())
-
-        return arcs
-
-
-class Chunk(object):
-    """A sequence of byte codes with a single entrance.
-
-    To analyze byte code, we have to divide it into chunks, sequences of byte
-    codes such that each chunk has only one entrance, the first instruction in
-    the block.
-
-    This is almost the CS concept of `basic block`_, except that we're willing
-    to have many exits from a chunk, and "basic block" is a more cumbersome
-    term.
-
-    .. _basic block: http://en.wikipedia.org/wiki/Basic_block
-
-    `byte` is the offset to the bytecode starting this chunk.
-
-    `line` is the source line number containing this chunk.
-
-    `first` is true if this is the first chunk in the source line.
-
-    An exit < 0 means the chunk can leave the code (return).  The exit is
-    the negative of the starting line number of the code block.
-
-    The `entrance` attribute is a boolean indicating whether the code object
-    can be entered at this chunk.
-
-    """
-    def __init__(self, byte, line, first):
-        self.byte = byte
-        self.line = line
-        self.first = first
-        self.length = 0
-        self.entrance = False
-        self.exits = set()
-
-    def __repr__(self):
-        return "<%d+%d @%d%s%s %r>" % (
-            self.byte,
-            self.length,
-            self.line,
-            "!" if self.first else "",
-            "v" if self.entrance else "",
-            list(self.exits),
-        )
-
 
 SKIP_DUMP_FIELDS = ["ctx"]
 
-- 
cgit v1.2.1


From cefd14cafc49a244c865885c87f019217d6d3a2f Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 7 Jan 2016 08:46:35 -0500
Subject: Bytecode not byte code

--HG--
branch : ast-branch
---
 coverage/parser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 32a75900..16419ca4 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -142,7 +142,7 @@ class PythonParser(object):
                 indent -= 1
             elif toktype == token.NAME:
                 if ttext == 'class':
-                    # Class definitions look like branches in the byte code, so
+                    # Class definitions look like branches in the bytecode, so
                     # we need to exclude them.  The simplest way is to note the
                     # lines with the 'class' keyword.
                     self.raw_classdefs.add(slineno)
@@ -720,7 +720,7 @@ class AstArcAnalyzer(object):
 
 
 class ByteParser(object):
-    """Parse byte codes to understand the structure of code."""
+    """Parse bytecode to understand the structure of code."""
 
     @contract(text='unicode')
     def __init__(self, text, code=None, filename=None):
-- 
cgit v1.2.1


From 152dd7d6e4b9a53e89cb7ec0cacf0f01be4abc73 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 7 Jan 2016 12:06:11 -0500
Subject: Clean up small stuff

--HG--
branch : ast-branch
---
 coverage/parser.py | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index 16419ca4..c03a3083 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -253,6 +253,12 @@ class PythonParser(object):
         self.statements = self.first_lines(starts) - ignore
 
     def arcs(self):
+        """Get information about the arcs available in the code.
+
+        Returns a set of line number pairs.  Line numbers have been normalized
+        to the first line of multi-line statements.
+
+        """
         if self._all_arcs is None:
             aaa = AstArcAnalyzer(self.text, self.raw_funcdefs, self.raw_classdefs)
             arcs = aaa.collect_arcs()
@@ -298,10 +304,12 @@ class LoopBlock(object):
         self.start = start
         self.break_exits = set()
 
+
 class FunctionBlock(object):
     def __init__(self, start):
         self.start = start
 
+
 class TryBlock(object):
     def __init__(self, handler_start=None, final_start=None):
         self.handler_start = handler_start  # TODO: is this used?
@@ -803,6 +811,7 @@ def is_simple_value(value):
         isinstance(value, (string_class, int, float))
     )
 
+# TODO: a test of ast_dump?
 def ast_dump(node, depth=0):
     indent = " " * depth
     if not isinstance(node, ast.AST):
-- 
cgit v1.2.1


From 2e48dedf1ea439988fba0c9693cea7a818ab3213 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 7 Jan 2016 19:42:42 -0500
Subject: Add tests of multiline lambdas, though i don't quite understand the
 line numbers involved

--HG--
branch : ast-branch
---
 coverage/parser.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'coverage/parser.py')

diff --git a/coverage/parser.py b/coverage/parser.py
index c03a3083..9f7400e5 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -312,8 +312,8 @@ class FunctionBlock(object):
 
 class TryBlock(object):
     def __init__(self, handler_start=None, final_start=None):
-        self.handler_start = handler_start  # TODO: is this used?
-        self.final_start = final_start      # TODO: is this used?
+        self.handler_start = handler_start
+        self.final_start = final_start
         self.break_from = set()
         self.continue_from = set()
         self.return_from = set()
@@ -716,7 +716,6 @@ class AstArcAnalyzer(object):
         start = self.line_for_node(node)
         self.arcs.add((-1, start))
         self.arcs.add((start, -start))
-        # TODO: test multi-line lambdas
 
     def contains_return_expression(self, node):
         """Is there a yield-from or await in `node` someplace?"""
-- 
cgit v1.2.1