* Merge changes from head.

1 files changed, 74 insertions, 211 deletions

diff --git a/coverage/collector.py b/coverage/collector.py
index 546525d2..85c8dc90 100644
--- a/coverage/collector.py
+++ b/coverage/collector.py
@@ -1,6 +1,9 @@
 """Raw data collector for Coverage."""
 
-import collections, os, sys, threading
+import os, sys
+
+from coverage.misc import CoverageException
+from coverage.pytracer import PyTracer
 
 try:
     # Use the C extension code when we can, for speed.
@@ -21,188 +24,6 @@ except ImportError:
     CTracer = None
 
 
-class PyTracer(object):
-    """Python implementation of the raw data tracer."""
-
-    # Because of poor implementations of trace-function-manipulating tools,
-    # the Python trace function must be kept very simple.  In particular, there
-    # must be only one function ever set as the trace function, both through
-    # sys.settrace, and as the return value from the trace function.  Put
-    # another way, the trace function must always return itself.  It cannot
-    # swap in other functions, or return None to avoid tracing a particular
-    # frame.
-    #
-    # The trace manipulator that introduced this restriction is DecoratorTools,
-    # which sets a trace function, and then later restores the pre-existing one
-    # by calling sys.settrace with a function it found in the current frame.
-    #
-    # Systems that use DecoratorTools (or similar trace manipulations) must use
-    # PyTracer to get accurate results.  The command-line --timid argument is
-    # used to force the use of this tracer.
-
-    def __init__(self):
-        # Attributes set from the collector:
-        self.data = None
-        self.arcs = False
-        self.should_trace = None
-        self.should_trace_cache = None
-        self.warn = None
-        self.extensions = None
-
-        self.extension = None
-        self.cur_tracename = None   # TODO: This is only maintained for the if0 debugging output. Get rid of it eventually.
-        self.cur_file_data = None
-        self.last_line = 0
-        self.data_stack = []
-        self.data_stacks = collections.defaultdict(list)
-        self.last_exc_back = None
-        self.last_exc_firstlineno = 0
-        self.thread = None
-        self.stopped = False
-        self.coroutine_id_func = None
-        self.last_coroutine = None
-
-    def _trace(self, frame, event, arg_unused):
-        """The trace function passed to sys.settrace."""
-
-        if self.stopped:
-            return
-
-        if 0:
-            # A lot of debugging to try to understand why gevent isn't right.
-            import os.path, pprint
-            def short_ident(ident):
-                return "{}:{:06X}".format(ident.__class__.__name__, id(ident) & 0xFFFFFF)
-
-            ident = None
-            if self.coroutine_id_func:
-                ident = short_ident(self.coroutine_id_func())
-            sys.stdout.write("trace event: %s %s %r @%d\n" % (
-                event, ident, frame.f_code.co_filename, frame.f_lineno
-            ))
-            pprint.pprint(
-                dict(
-                    (
-                        short_ident(ident),
-                        [
-                            (os.path.basename(tn or ""), sorted((cfd or {}).keys()), ll)
-                            for ex, tn, cfd, ll in data_stacks
-                        ]
-                    )
-                    for ident, data_stacks in self.data_stacks.items()
-                )
-                , width=250)
-            pprint.pprint(sorted((self.cur_file_data or {}).keys()), width=250)
-            print("TRYING: {}".format(sorted(next((v for k,v in self.data.items() if k.endswith("try_it.py")), {}).keys())))
-
-        if self.last_exc_back:
-            if frame == self.last_exc_back:
-                # Someone forgot a return event.
-                if self.arcs and self.cur_file_data:
-                    pair = (self.last_line, -self.last_exc_firstlineno)
-                    self.cur_file_data[pair] = None
-                if self.coroutine_id_func:
-                    self.data_stack = self.data_stacks[self.coroutine_id_func()]
-                self.handler, _, self.cur_file_data, self.last_line = self.data_stack.pop()
-            self.last_exc_back = None
-
-        if event == 'call':
-            # Entering a new function context.  Decide if we should trace
-            # in this file.
-            if self.coroutine_id_func:
-                self.data_stack = self.data_stacks[self.coroutine_id_func()]
-                self.last_coroutine = self.coroutine_id_func()
-            self.data_stack.append((self.extension, self.cur_tracename, self.cur_file_data, self.last_line))
-            filename = frame.f_code.co_filename
-            disp = self.should_trace_cache.get(filename)
-            if disp is None:
-                disp = self.should_trace(filename, frame)
-                self.should_trace_cache[filename] = disp
-            #print("called, stack is %d deep, tracename is %r" % (
-            #               len(self.data_stack), tracename))
-            tracename = disp.filename
-            if tracename and disp.extension:
-                tracename = disp.extension.file_name(frame)
-            if tracename:
-                if tracename not in self.data:
-                    self.data[tracename] = {}
-                    if disp.extension:
-                        self.extensions[tracename] = disp.extension.__name__
-                self.cur_tracename = tracename
-                self.cur_file_data = self.data[tracename]
-                self.extension = disp.extension
-            else:
-                self.cur_file_data = None
-            # Set the last_line to -1 because the next arc will be entering a
-            # code block, indicated by (-1, n).
-            self.last_line = -1
-        elif event == 'line':
-            # Record an executed line.
-            if 0 and self.coroutine_id_func:
-                this_coroutine = self.coroutine_id_func()
-                if self.last_coroutine != this_coroutine:
-                    print("mismatch: {0} != {1}".format(self.last_coroutine, this_coroutine))
-            if self.extension:
-                lineno_from, lineno_to = self.extension.line_number_range(frame)
-            else:
-                lineno_from, lineno_to = frame.f_lineno, frame.f_lineno
-            if lineno_from != -1:
-                if self.cur_file_data is not None:
-                    if self.arcs:
-                        #print("lin", self.last_line, frame.f_lineno)
-                        self.cur_file_data[(self.last_line, lineno_from)] = None
-                    else:
-                        #print("lin", frame.f_lineno)
-                        for lineno in range(lineno_from, lineno_to+1):
-                            self.cur_file_data[lineno] = None
-                self.last_line = lineno_to
-        elif event == 'return':
-            if self.arcs and self.cur_file_data:
-                first = frame.f_code.co_firstlineno
-                self.cur_file_data[(self.last_line, -first)] = None
-            # Leaving this function, pop the filename stack.
-            if self.coroutine_id_func:
-                self.data_stack = self.data_stacks[self.coroutine_id_func()]
-                self.last_coroutine = self.coroutine_id_func()
-            self.extension, _, self.cur_file_data, self.last_line = self.data_stack.pop()
-            #print("returned, stack is %d deep" % (len(self.data_stack)))
-        elif event == 'exception':
-            #print("exc", self.last_line, frame.f_lineno)
-            self.last_exc_back = frame.f_back
-            self.last_exc_firstlineno = frame.f_code.co_firstlineno
-        return self._trace
-
-    def start(self):
-        """Start this Tracer.
-
-        Return a Python function suitable for use with sys.settrace().
-
-        """
-        self.thread = threading.currentThread()
-        sys.settrace(self._trace)
-        return self._trace
-
-    def stop(self):
-        """Stop this Tracer."""
-        self.stopped = True
-        if self.thread != threading.currentThread():
-            # Called on a different thread than started us: we can't unhook
-            # ourseves, but we've set the flag that we should stop, so we won't
-            # do any more tracing.
-            return
-
-        if hasattr(sys, "gettrace") and self.warn:
-            if sys.gettrace() != self._trace:
-                msg = "Trace function changed, measurement is likely wrong: %r"
-                self.warn(msg % (sys.gettrace(),))
-        #print("Stopping tracer on %s" % threading.current_thread().ident)
-        sys.settrace(None)
-
-    def get_stats(self):
-        """Return a dictionary of statistics, or None."""
-        return None
-
-
 class Collector(object):
     """Collects trace data.
 
@@ -224,13 +45,17 @@ class Collector(object):
     # the top, and resumed when they become the top again.
     _collectors = []
 
-    def __init__(self, should_trace, timid, branch, warn, coroutine):
+    def __init__(self,
+        should_trace, check_include, timid, branch, warn, coroutine,
+    ):
         """Create a collector.
 
         `should_trace` is a function, taking a filename, and returning a
         canonicalized filename, or None depending on whether the file should
         be traced or not.
 
+        TODO: `check_include`
+
         If `timid` is true, then a slower simpler trace function will be
         used.  This is important for some environments where manipulation of
         tracing functions make the faster more sophisticated trace function not
@@ -243,21 +68,44 @@ class Collector(object):
         `warn` is a warning function, taking a single string message argument,
         to be used if a warning needs to be issued.
 
+        TODO: `coroutine`
+
         """
         self.should_trace = should_trace
+        self.check_include = check_include
         self.warn = warn
         self.branch = branch
-        if coroutine == "greenlet":
-            import greenlet
-            self.coroutine_id_func = greenlet.getcurrent
-        elif coroutine == "eventlet":
-            import eventlet.greenthread
-            self.coroutine_id_func = eventlet.greenthread.getcurrent
-        elif coroutine == "gevent":
-            import gevent
-            self.coroutine_id_func = gevent.getcurrent
-        else:
-            self.coroutine_id_func = None
+        self.threading = None
+        self.coroutine = coroutine
+
+        self.coroutine_id_func = None
+
+        try:
+            if coroutine == "greenlet":
+                import greenlet
+                self.coroutine_id_func = greenlet.getcurrent
+            elif coroutine == "eventlet":
+                import eventlet.greenthread
+                self.coroutine_id_func = eventlet.greenthread.getcurrent
+            elif coroutine == "gevent":
+                import gevent
+                self.coroutine_id_func = gevent.getcurrent
+            elif coroutine == "thread" or not coroutine:
+                # It's important to import threading only if we need it.  If
+                # it's imported early, and the program being measured uses
+                # gevent, then gevent's monkey-patching won't work properly.
+                import threading
+                self.threading = threading
+            else:
+                raise CoverageException(
+                    "Don't understand coroutine=%s" % coroutine
+                )
+        except ImportError:
+            raise CoverageException(
+                "Couldn't trace with coroutine=%s, "
+                "the module isn't installed." % coroutine
+            )
+
         self.reset()
 
         if timid:
@@ -281,7 +129,7 @@ class Collector(object):
         # or mapping filenames to dicts with linenumber pairs as keys.
         self.data = {}
 
-        self.extensions = {}
+        self.plugin_data = {}
 
         # A cache of the results from should_trace, the decision about whether
         # to trace execution in a file. A dict of filename to (filename or
@@ -299,12 +147,25 @@ class Collector(object):
         tracer.should_trace = self.should_trace
         tracer.should_trace_cache = self.should_trace_cache
         tracer.warn = self.warn
+
         if hasattr(tracer, 'coroutine_id_func'):
             tracer.coroutine_id_func = self.coroutine_id_func
-        if hasattr(tracer, 'extensions'):
-            tracer.extensions = self.extensions
+        elif self.coroutine_id_func:
+            raise CoverageException(
+                "Can't support coroutine=%s with %s, "
+                "only threads are supported" % (
+                    self.coroutine, self.tracer_name(),
+                )
+            )
+
+        if hasattr(tracer, 'plugin_data'):
+            tracer.plugin_data = self.plugin_data
+        if hasattr(tracer, 'threading'):
+            tracer.threading = self.threading
+
         fn = tracer.start()
         self.tracers.append(tracer)
+
         return fn
 
     # The trace function has to be set individually on each thread before
@@ -331,16 +192,14 @@ class Collector(object):
         if self._collectors:
             self._collectors[-1].pause()
         self._collectors.append(self)
-        #print("Started: %r" % self._collectors, file=sys.stderr)
 
         # Check to see whether we had a fullcoverage tracer installed.
         traces0 = []
-        if hasattr(sys, "gettrace"):
-            fn0 = sys.gettrace()
-            if fn0:
-                tracer0 = getattr(fn0, '__self__', None)
-                if tracer0:
-                    traces0 = getattr(tracer0, 'traces', [])
+        fn0 = sys.gettrace()
+        if fn0:
+            tracer0 = getattr(fn0, '__self__', None)
+            if tracer0:
+                traces0 = getattr(tracer0, 'traces', [])
 
         # Install the tracer on this thread.
         fn = self._start_tracer()
@@ -356,11 +215,11 @@ class Collector(object):
 
         # Install our installation tracer in threading, to jump start other
         # threads.
-        threading.settrace(self._installation_trace)
+        if self.threading:
+            self.threading.settrace(self._installation_trace)
 
     def stop(self):
         """Stop collecting trace information."""
-        #print >>sys.stderr, "Stopping: %r" % self._collectors
         assert self._collectors
         assert self._collectors[-1] is self
 
@@ -382,13 +241,17 @@ class Collector(object):
                 print("\nCoverage.py tracer stats:")
                 for k in sorted(stats.keys()):
                     print("%16s: %s" % (k, stats[k]))
-        threading.settrace(None)
+        if self.threading:
+            self.threading.settrace(None)
 
     def resume(self):
         """Resume tracing after a `pause`."""
         for tracer in self.tracers:
             tracer.start()
-        threading.settrace(self._installation_trace)
+        if self.threading:
+            self.threading.settrace(self._installation_trace)
+        else:
+            self._start_tracer()
 
     def get_line_data(self):
         """Return the line data collected.
@@ -420,5 +283,5 @@ class Collector(object):
         else:
             return {}
 
-    def get_extension_data(self):
-        return self.extensions
+    def get_plugin_data(self):
+        return self.plugin_data