Merge branch 'nedbat/dynamic-contexts'

10 files changed, 220 insertions, 50 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 089fec7f..b449797b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -19,8 +19,27 @@ Unreleased
 
 - Context support: static contexts let you specify a label for a coverage run,
   which is recorded in the data, and retained when you combine files.  See
-  :ref:`contexts` for more information.  Currently, only static contexts are
-  supported, with no reporting features.
+  :ref:`contexts` for more information.
+
+- Dynamic contexts: specifying ``[run] dynamic_context = test_function`` in the
+  config file will record the test function name as a dynamic context during
+  execution.  This is the core of "Who Tests What" (`issue 170`_).  Things to
+  note:
+
+  - There is no reporting support yet.  Use SQLite to query the .coverage file
+    for information.  Ideas are welcome about how reporting could be extended
+    to use this data.
+
+  - There's a noticeable slow-down before any test is run.
+
+  - Data files will now be roughly N times larger, where N is the number of
+    tests you have.  Combining data files is therefore also N times slower.
+
+  - No other values for ``dynamic_context`` are recognized yet.  Let me know
+    what else would be useful.  I'd like to use a pytest plugin to get better
+    information directly from pytest, for example.
+
+.. _issue 170: https://github.com/nedbat/coveragepy/issues/170
 
 - Environment variable substitution in configuration files now supports two
   syntaxes for controlling the behavior of undefined variables: if ``VARNAME``
diff --git a/coverage/collector.py b/coverage/collector.py
index e0144979..4e7058a0 100644
--- a/coverage/collector.py
+++ b/coverage/collector.py
@@ -34,14 +34,6 @@ except ImportError:
     CTracer = None
 
 
-def should_start_context(frame):
-    """Who-Tests-What hack: Determine whether this frame begins a new who-context."""
-    fn_name = frame.f_code.co_name
-    if fn_name.startswith("test"):
-        return fn_name
-    return None
-
-
 class Collector(object):
     """Collects trace data.
 
@@ -66,7 +58,10 @@ class Collector(object):
     # The concurrency settings we support here.
     SUPPORTED_CONCURRENCIES = set(["greenlet", "eventlet", "gevent", "thread"])
 
-    def __init__(self, should_trace, check_include, timid, branch, warn, concurrency):
+    def __init__(
+        self, should_trace, check_include, should_start_context,
+        timid, branch, warn, concurrency,
+    ):
         """Create a collector.
 
         `should_trace` is a function, taking a file name and a frame, and
@@ -75,6 +70,11 @@ class Collector(object):
         `check_include` is a function taking a file name and a frame. It returns
         a boolean: True if the file should be traced, False if not.
 
+        `should_start_context` is a function taking a frame, and returning a
+        string. If the frame should be the start of a new context, the string
+        is the new context. If the frame should not be the start of a new
+        context, return None.
+
         If `timid` is true, then a slower simpler trace function will be
         used.  This is important for some environments where manipulation of
         tracing functions make the faster more sophisticated trace function not
@@ -96,6 +96,7 @@ class Collector(object):
         """
         self.should_trace = should_trace
         self.check_include = check_include
+        self.should_start_context = should_start_context
         self.warn = warn
         self.branch = branch
         self.threading = None
@@ -139,10 +140,6 @@ class Collector(object):
                 )
             )
 
-        # Who-Tests-What is just a hack at the moment, so turn it on with an
-        # environment variable.
-        self.wtw = int(os.getenv('COVERAGE_WTW', 0))
-
         self.reset()
 
         if timid:
@@ -175,7 +172,11 @@ class Collector(object):
 
     def _clear_data(self):
         """Clear out existing data, but stay ready for more collection."""
-        self.data.clear()
+        # We used to used self.data.clear(), but that would remove filename
+        # keys and data values that were still in use higher up the stack
+        # when we are called as part of switch_context.
+        for d in self.data.values():
+            d.clear()
 
         for tracer in self.tracers:
             tracer.reset_activity()
@@ -187,10 +188,6 @@ class Collector(object):
         # pairs as keys (if branch coverage).
         self.data = {}
 
-        # A dict mapping contexts to data dictionaries.
-        self.contexts = {}
-        self.contexts[None] = self.data
-
         # A dictionary mapping file names to file tracer plugin names that will
         # handle them.
         self.file_tracers = {}
@@ -252,11 +249,13 @@ class Collector(object):
             tracer.threading = self.threading
         if hasattr(tracer, 'check_include'):
             tracer.check_include = self.check_include
-        if self.wtw:
-            if hasattr(tracer, 'should_start_context'):
-                tracer.should_start_context = should_start_context
-            if hasattr(tracer, 'switch_context'):
-                tracer.switch_context = self.switch_context
+        if hasattr(tracer, 'should_start_context'):
+            tracer.should_start_context = self.should_start_context
+            tracer.switch_context = self.switch_context
+        elif self.should_start_context:
+            raise CoverageException(
+                "Can't support dynamic contexts with {}".format(self.tracer_name())
+            )
 
         fn = tracer.start()
         self.tracers.append(tracer)
@@ -372,12 +371,15 @@ class Collector(object):
         return any(tracer.activity() for tracer in self.tracers)
 
     def switch_context(self, new_context):
-        """Who-Tests-What hack: switch to a new who-context."""
-        # Make a new data dict, or find the existing one, and switch all the
-        # tracers to use it.
-        data = self.contexts.setdefault(new_context, {})
-        for tracer in self.tracers:
-            tracer.data = data
+        """Switch to a new dynamic context."""
+        self.flush_data()
+        if self.static_context:
+            context = self.static_context
+            if new_context:
+                context += ":" + new_context
+        else:
+            context = new_context
+        self.covdata.set_context(context)
 
     def cached_abs_file(self, filename):
         """A locally cached version of `abs_file`."""
@@ -415,7 +417,7 @@ class Collector(object):
             else:
                 raise runtime_err       # pylint: disable=raising-bad-type
 
-            return dict((self.cached_abs_file(k), v) for k, v in items)
+            return dict((self.cached_abs_file(k), v) for k, v in items if v)
 
         if self.branch:
             self.covdata.add_arcs(abs_file_dict(self.data))
@@ -423,12 +425,5 @@ class Collector(object):
             self.covdata.add_lines(abs_file_dict(self.data))
         self.covdata.add_file_tracers(abs_file_dict(self.file_tracers))
 
-        if self.wtw:
-            # Just a hack, so just hack it.
-            import pprint
-            out_file = "coverage_wtw_{:06}.py".format(os.getpid())
-            with open(out_file, "w") as wtw_out:
-                pprint.pprint(self.contexts, wtw_out)
-
         self._clear_data()
         return True
diff --git a/coverage/config.py b/coverage/config.py
index 9a11323d..2a281875 100644
--- a/coverage/config.py
+++ b/coverage/config.py
@@ -180,6 +180,7 @@ class CoverageConfig(object):
         self.data_file = ".coverage"
         self.debug = []
         self.disable_warnings = []
+        self.dynamic_context = None
         self.note = None
         self.parallel = False
         self.plugins = []
@@ -324,6 +325,7 @@ class CoverageConfig(object):
         ('data_file', 'run:data_file'),
         ('debug', 'run:debug', 'list'),
         ('disable_warnings', 'run:disable_warnings', 'list'),
+        ('dynamic_context', 'run:dynamic_context'),
         ('note', 'run:note'),
         ('parallel', 'run:parallel', 'boolean'),
         ('plugins', 'run:plugins', 'list'),
diff --git a/coverage/control.py b/coverage/control.py
index 678a7b3b..4f2afda0 100644
--- a/coverage/control.py
+++ b/coverage/control.py
@@ -347,9 +347,19 @@ class Coverage(object):
             # it for the main process.
             self.config.parallel = True
 
+        if self.config.dynamic_context is None:
+            should_start_context = None
+        elif self.config.dynamic_context == "test_function":
+            should_start_context = should_start_context_test_function
+        else:
+            raise CoverageException(
+                "Don't understand dynamic_context setting: {!r}".format(self.config.dynamic_context)
+            )
+
         self._collector = Collector(
             should_trace=self._should_trace,
             check_include=self._check_include_omit_etc,
+            should_start_context=should_start_context,
             timid=self.config.timid,
             branch=self.config.branch,
             warn=self._warn,
@@ -886,6 +896,16 @@ if int(os.environ.get("COVERAGE_DEBUG_CALLS", 0)):              # pragma: debugg
     Coverage = decorate_methods(show_calls(show_args=True), butnot=['get_data'])(Coverage)
 
 
+def should_start_context_test_function(frame):
+    """Who-Tests-What hack: Determine whether this frame begins a new who-context."""
+    with open("/tmp/ssc.txt", "a") as f:
+        f.write("hello\n")
+    fn_name = frame.f_code.co_name
+    if fn_name.startswith("test"):
+        return fn_name
+    return None
+
+
 def process_startup():
     """Call this at Python start-up to perhaps measure coverage.
 
diff --git a/coverage/ctracer/tracer.c b/coverage/ctracer/tracer.c
index 01f8b19b..7d639112 100644
--- a/coverage/ctracer/tracer.c
+++ b/coverage/ctracer/tracer.c
@@ -341,7 +341,6 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame)
     CFileDisposition * pdisp = NULL;
 
     STATS( self->stats.calls++; )
-    self->activity = TRUE;
 
     /* Grow the stack. */
     if (CTracer_set_pdata_stack(self) < 0) {
@@ -353,7 +352,7 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame)
     self->pcur_entry = &self->pdata_stack->stack[self->pdata_stack->depth];
 
     /* See if this frame begins a new context. */
-    if (self->should_start_context && self->context == Py_None) {
+    if (self->should_start_context != Py_None && self->context == Py_None) {
         PyObject * context;
         /* We're looking for our context, ask should_start_context if this is the start. */
         STATS( self->stats.start_context_calls++; )
@@ -866,6 +865,8 @@ CTracer_trace(CTracer *self, PyFrameObject *frame, int what, PyObject *arg_unuse
         goto error;
     }
 
+    self->activity = TRUE;
+
     switch (what) {
     case PyTrace_CALL:
         if (CTracer_handle_call(self, frame) < 0) {
diff --git a/coverage/ctracer/tracer.h b/coverage/ctracer/tracer.h
index 61c01b41..a83742dd 100644
--- a/coverage/ctracer/tracer.h
+++ b/coverage/ctracer/tracer.h
@@ -27,7 +27,6 @@ typedef struct CTracer {
     PyObject * trace_arcs;
     PyObject * should_start_context;
     PyObject * switch_context;
-    PyObject * context;
 
     /* Has the tracer been started? */
     BOOL started;
@@ -35,6 +34,8 @@ typedef struct CTracer {
     BOOL tracing_arcs;
     /* Have we had any activity? */
     BOOL activity;
+    /* The current dynamic context. */
+    PyObject * context;
 
     /*
         The data stack is a stack of dictionaries.  Each dictionary collects
diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index fb2279c9..738fccef 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -199,6 +199,8 @@ class CoverageSqliteData(SimpleReprMixin):
 
     def set_context(self, context):
         """Set the current context for future `add_lines` etc."""
+        if self._debug and self._debug.should('dataop'):
+            self._debug.write("Setting context: %r" % (context,))
         self._start_using()
         context = context or ""
         with self._connect() as con:
diff --git a/doc/contexts.rst b/doc/contexts.rst
index c1d4a173..1f1ce763 100644
--- a/doc/contexts.rst
+++ b/doc/contexts.rst
@@ -16,9 +16,8 @@ in which it was run.  This can provide more information to help you understand
 the behavior of your tests.
 
 There are two kinds of context: static and dynamic.  Static contexts are fixed
-for an entire run, and are set explicitly with an option.
-
-Dynamic contexts are coming soon.
+for an entire run, and are set explicitly with an option.  Dynamic contexts
+change over the course of a single run.
 
 
 Static contexts
@@ -39,7 +38,18 @@ A static context is specified with the ``--context=CONTEXT`` option to
 Dynamic contexts
 ----------------
 
-Not implemented yet.
+Dynamic contexts are found during execution.  There is currently support for
+one kind: test function names.  Set the ``dynamic_context`` option to
+``test_function`` in the configuration file::
+
+    [run]
+    dynamic_context = test_function
+
+Each test function you run will be considered a separate dynamic context, and
+coverage data will be segregated for each.  A test function is any function
+whose names starts with "test".
+
+Ideas are welcome for other dynamic contexts that would be useful.
 
 
 Context reporting
diff --git a/lab/find_class.py b/lab/find_class.py
new file mode 100644
index 00000000..d8dac0b5
--- /dev/null
+++ b/lab/find_class.py
@@ -0,0 +1,40 @@
+class Parent(object):
+    def meth(self):
+        print("METH")
+
+class Child(Parent):
+    pass
+
+def trace(frame, event, args):
+    # Thanks to Aleksi Torhamo for code and idea.
+    co = frame.f_code
+    fname = co.co_name
+    if not co.co_varnames:
+        return
+    locs = frame.f_locals
+    first_arg = co.co_varnames[0]
+    if co.co_argcount:
+        self = locs[first_arg]
+    elif co.co_flags & 0x04:    # *args syntax
+        self = locs[first_arg][0]
+    else:
+        return
+
+    func = getattr(self, fname).__func__
+    if hasattr(func, '__qualname__'):
+        qname = func.__qualname__
+    else:
+        for cls in self.__class__.__mro__:
+            f = cls.__dict__.get(fname, None)
+            if f is None:
+                continue
+            if f is func:
+                qname = cls.__name__ + "." + fname
+                break
+    print("{}: {}.{} {}".format(event, self, fname, qname))
+    return trace
+
+import sys
+sys.settrace(trace)
+
+Child().meth()
diff --git a/tests/test_context.py b/tests/test_context.py
index a6be922d..4339d336 100644
--- a/tests/test_context.py
+++ b/tests/test_context.py
@@ -6,16 +6,18 @@
 import os.path
 
 import coverage
+from coverage import env
 from coverage.data import CoverageData
+from coverage.misc import CoverageException
 
 from tests.coveragetest import CoverageTest
 
 
-class GlobalContextTest(CoverageTest):
-    """Tests of the global context."""
+class StaticContextTest(CoverageTest):
+    """Tests of the static context."""
 
     def setUp(self):
-        super(GlobalContextTest, self).setUp()
+        super(StaticContextTest, self).setUp()
         self.skip_unless_data_storage_is("sql")
 
     def test_no_context(self):
@@ -25,7 +27,7 @@ class GlobalContextTest(CoverageTest):
         data = cov.get_data()
         self.assertCountEqual(data.measured_contexts(), [""])
 
-    def test_global_context(self):
+    def test_static_context(self):
         self.make_file("main.py", "a = 1")
         cov = coverage.Coverage(context="gooey")
         self.start_import_stop(cov, "main")
@@ -102,3 +104,81 @@ class GlobalContextTest(CoverageTest):
             self.assertEqual(combined.arcs(fred, context='blue'), [])
             self.assertEqual(combined.arcs(fblue, context='red'), [])
             self.assertEqual(combined.arcs(fblue, context='blue'), self.ARCS)
+
+
+class DynamicContextTest(CoverageTest):
+    """Tests of dynamically changing contexts."""
+
+    def setUp(self):
+        super(DynamicContextTest, self).setUp()
+        self.skip_unless_data_storage_is("sql")
+        if not env.C_TRACER:
+            self.skipTest("Only the C tracer supports dynamic contexts")
+
+    SOURCE = """\
+        def helper(lineno):
+            x = 2
+
+        def test_one():
+            a = 5
+            helper(6)
+
+        def test_two():
+            a = 9
+            b = 10
+            if a > 11:
+                b = 12
+            assert a == (13-4)
+            assert b == (14-4)
+            helper(15)
+
+        test_one()
+        x = 18
+        helper(19)
+        test_two()
+        """
+
+    OUTER_LINES = [1, 4, 8, 17, 18, 19, 2, 20]
+    TEST_ONE_LINES = [5, 6, 2]
+    TEST_TWO_LINES = [9, 10, 11, 13, 14, 15, 2]
+
+    def test_dynamic_alone(self):
+        self.make_file("two_tests.py", self.SOURCE)
+        cov = coverage.Coverage(source=["."])
+        cov.set_option("run:dynamic_context", "test_function")
+        self.start_import_stop(cov, "two_tests")
+        data = cov.get_data()
+
+        fname = os.path.abspath("two_tests.py")
+        self.assertCountEqual(data.measured_contexts(), ["", "test_one", "test_two"])
+        self.assertCountEqual(data.lines(fname, ""), self.OUTER_LINES)
+        self.assertCountEqual(data.lines(fname, "test_one"), self.TEST_ONE_LINES)
+        self.assertCountEqual(data.lines(fname, "test_two"), self.TEST_TWO_LINES)
+
+    def test_static_and_dynamic(self):
+        self.make_file("two_tests.py", self.SOURCE)
+        cov = coverage.Coverage(context="stat", source=["."])
+        cov.set_option("run:dynamic_context", "test_function")
+        self.start_import_stop(cov, "two_tests")
+        data = cov.get_data()
+
+        fname = os.path.abspath("two_tests.py")
+        self.assertCountEqual(data.measured_contexts(), ["stat", "stat:test_one", "stat:test_two"])
+        self.assertCountEqual(data.lines(fname, "stat"), self.OUTER_LINES)
+        self.assertCountEqual(data.lines(fname, "stat:test_one"), self.TEST_ONE_LINES)
+        self.assertCountEqual(data.lines(fname, "stat:test_two"), self.TEST_TWO_LINES)
+
+
+class DynamicContextWithPythonTracerTest(CoverageTest):
+    """The Python tracer doesn't do dynamic contexts at all."""
+
+    run_in_temp_dir = False
+
+    def test_python_tracer_fails_properly(self):
+        if env.C_TRACER:
+            self.skipTest("This test is specifically about the Python tracer.")
+        cov = coverage.Coverage()
+        cov.set_option("run:dynamic_context", "test_function")
+        msg = r"Can't support dynamic contexts with PyTracer"
+        with self.assertRaisesRegex(CoverageException, msg):
+            cov.start()