Plugin support for dynamic context (#783)

* Introduce a new plugin type: dynamic context labels.

* Test dynamic context plugins

* Helper method to get full paths to measured files

* Get correct filenames on all OS

* Improve wording

5 files changed, 335 insertions, 3 deletions

diff --git a/coverage/control.py b/coverage/control.py
index ea6698d4..ae5f0442 100644
--- a/coverage/control.py
+++ b/coverage/control.py
@@ -335,6 +335,34 @@ class Coverage(object):
         if not should_skip:
             self._data.read()
 
+    def _combine_context_switchers(self, context_switchers):
+        """Create a single context switcher from multiple switchers.
+
+        `context_switchers` is a list of methods that take a frame
+        as an argument and return a string to use as the new context label.
+
+        Returns a method that composits `context_switchers` methods, or None
+        if `context_switchers` is an empty list.
+
+        When invoked, the combined switcher calls `context_switchers` one-by-one
+        until a string is returned.  Combined switcher returns None if all
+        `context_switchers` return None.
+        """
+        if not context_switchers:
+            return None
+
+        if len(context_switchers) == 1:
+            return context_switchers[0]
+
+        def should_start_context(frame):
+            for switcher in context_switchers:
+                new_context = switcher(frame)
+                if new_context is not None:
+                    return new_context
+            return None
+
+        return should_start_context
+
     def _init_for_start(self):
         """Initialization for start()"""
         # Construct the collector.
@@ -350,14 +378,21 @@ class Coverage(object):
             self.config.parallel = True
 
         if self.config.dynamic_context is None:
-            should_start_context = None
+            context_switchers = []
         elif self.config.dynamic_context == "test_function":
-            should_start_context = should_start_context_test_function
+            context_switchers = [should_start_context_test_function]
         else:
             raise CoverageException(
                 "Don't understand dynamic_context setting: {!r}".format(self.config.dynamic_context)
             )
 
+        context_switchers.extend([
+            plugin.dynamic_context
+            for plugin in self._plugins.context_switchers
+            ])
+
+        should_start_context = self._combine_context_switchers(context_switchers)
+
         self._collector = Collector(
             should_trace=self._should_trace,
             check_include=self._check_include_omit_etc,
diff --git a/coverage/plugin.py b/coverage/plugin.py
index f65d419c..e817cf2f 100644
--- a/coverage/plugin.py
+++ b/coverage/plugin.py
@@ -78,6 +78,26 @@ change the configuration.
 In your ``coverage_init`` function, use the ``add_configurer`` method to
 register your configurer.
 
+Dynamic Contexts
+================
+
+.. versionadded:: 5.0
+
+Context plugins implement the :meth:`~coverage.CoveragePlugin.dynamic_context` method
+to dynamically compute the context label for each measured frame.
+
+Computed context labels are useful when you want to group measured data without
+modifying the source code.
+
+For example, you could write a plugin that check `frame.f_code` to inspect
+the currently executed method, and set label to a fully qualified method
+name if it's an instance method of `unittest.TestCase` and the method name
+starts with 'test'.  Such plugin would provide basic coverage grouping by test
+and could be used with test runners that have no built-in coveragepy support.
+
+In your ``coverage_init`` function, use the ``add_dynamic_context`` method to
+register your file tracer.
+
 """
 
 from coverage import files
@@ -140,6 +160,17 @@ class CoveragePlugin(object):
         """
         _needs_to_implement(self, "file_reporter")
 
+    def dynamic_context(self, frame):       # pylint: disable=unused-argument
+        """Get dynamically computed context label for collected data.
+
+        Plug-in type: dynamic context.
+
+        This method is invoked for each frame.  If it returns a string,
+        a new context label is set for this and deeper frames.
+
+        """
+        return None
+
     def find_executable_files(self, src_dir):       # pylint: disable=unused-argument
         """Yield all of the executable files in `src_dir`, recursively.
 
diff --git a/coverage/plugin_support.py b/coverage/plugin_support.py
index 0727a3b0..7c25a5f1 100644
--- a/coverage/plugin_support.py
+++ b/coverage/plugin_support.py
@@ -21,6 +21,7 @@ class Plugins(object):
         self.names = {}
         self.file_tracers = []
         self.configurers = []
+        self.context_switchers = []
 
         self.current_module = None
         self.debug = None
@@ -70,6 +71,15 @@ class Plugins(object):
         """
         self._add_plugin(plugin, self.configurers)
 
+    def add_dynamic_context(self, plugin):
+        """Add a dynamic context plugin.
+
+        `plugin` is an instance of a third-party plugin class.  It must
+        implement the :meth:`CoveragePlugin.dynamic_context` method.
+
+        """
+        self._add_plugin(plugin, self.context_switchers)
+
     def add_noop(self, plugin):
         """Add a plugin that does nothing.
 
diff --git a/tests/coveragetest.py b/tests/coveragetest.py
index 98c1e087..c0a8fb5a 100644
--- a/tests/coveragetest.py
+++ b/tests/coveragetest.py
@@ -8,6 +8,7 @@ import datetime
 import functools
 import glob
 import os
+import os.path
 import random
 import re
 import shlex
@@ -514,6 +515,15 @@ class CoverageTest(
         """Return the last line of `report` with the spaces squeezed down."""
         return self.squeezed_lines(report)[-1]
 
+    def get_measured_filenames(self, coverage_data):
+        """Get paths to measured files.
+
+        Returns a dict of {filename: absolute path to file}
+        for given CoverageData.
+        """
+        return {os.path.basename(filename): filename
+                for filename in coverage_data.measured_files()}
+
 
 class UsingModulesMixin(object):
     """A mixin for importing modules from tests/modules and tests/moremodules."""
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 14d07c1a..74f301e3 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -3,12 +3,13 @@
 
 """Tests for plugins."""
 
+import inspect
 import os.path
 from xml.etree import ElementTree
 
 import coverage
 from coverage import env
-from coverage.backward import StringIO
+from coverage.backward import StringIO, import_local_file
 from coverage.data import line_counts
 from coverage.control import Plugins
 from coverage.misc import CoverageException
@@ -883,3 +884,248 @@ class ConfigurerPluginTest(CoverageTest):
         excluded = cov.get_option("report:exclude_lines")
         self.assertIn("pragma: custom", excluded)
         self.assertIn("pragma: or whatever", excluded)
+
+
+class DynamicContextPluginTest(CoverageTest):
+    """Tests of plugins that implement `dynamic_context`."""
+
+    def setUp(self):
+        super(DynamicContextPluginTest, self).setUp()
+        if not env.C_TRACER:
+            self.skipTest("Plugins are only supported with the C tracer.")
+
+    def make_plugin_capitalized_testnames(self, filename):
+        self.make_file(filename, """\
+            from coverage import CoveragePlugin
+
+            class Plugin(CoveragePlugin):
+                def dynamic_context(self, frame):
+                    name = frame.f_code.co_name
+                    if (name.startswith("test_")
+                            or name.startswith("doctest_")):
+                        parts = name.split("_", 1)
+                        return "%s:%s" % (parts[0], parts[1].upper())
+                    return None
+
+            def coverage_init(reg, options):
+                reg.add_dynamic_context(Plugin())
+            """)
+
+    def make_plugin_track_render(self, filename):
+        self.make_file(filename, """\
+            from coverage import CoveragePlugin
+
+            class Plugin(CoveragePlugin):
+                def dynamic_context(self, frame):
+                    name = frame.f_code.co_name
+                    if name.startswith("render_"):
+                        return 'renderer:' + name[7:]
+                    return None
+
+            def coverage_init(reg, options):
+                reg.add_dynamic_context(Plugin())
+            """)
+
+    def make_testsuite(self):
+        filenames = []
+        self.make_file("rendering.py", """\
+            def html_tag(tag, content):
+                return '<%s>%s</%s>' % (tag, content, tag)
+
+            def render_paragraph(text):
+                return html_tag('p', text)
+
+            def render_span(text):
+                return html_tag('span', text)
+
+            def render_bold(text):
+                return html_tag('b', text)
+                """)
+
+        self.make_file("testsuite.py", """\
+            import rendering
+
+            def test_html_tag():
+                assert rendering.html_tag('b', 'hello') == '<b>hello</b>'
+
+            def doctest_html_tag():
+                assert eval('''
+                    rendering.html_tag('i', 'text') == '<i>text</i>'
+                    '''.strip())
+
+            def test_renderers():
+                assert rendering.render_paragraph('hello') == '<p>hello</p>'
+                assert rendering.render_bold('wide') == '<b>wide</b>'
+                assert rendering.render_span('world') == '<span>world</span>'
+
+            def build_full_html():
+                html = '<html><body>%s</body></html>' % (
+                   rendering.render_paragraph(
+                      rendering.render_span('hello')))
+                return html
+            """)
+
+    def run_testsuite(self, coverage, suite_name):
+        coverage.start()
+        suite = import_local_file(suite_name)
+        try:
+            # Call all functions in this module
+            for name in dir(suite):
+                variable = getattr(suite, name)
+                if inspect.isfunction(variable):
+                    variable()
+        finally:
+            coverage.stop()
+        return suite
+
+    def test_plugin_standalone(self):
+        self.make_plugin_capitalized_testnames('plugin_tests.py')
+        self.make_testsuite()
+
+        # Enable dynamic context plugin
+        cov = coverage.Coverage()
+        cov.set_option("run:plugins", ['plugin_tests'])
+
+        # Run the tests
+        suite = self.run_testsuite(cov, 'testsuite')
+
+        # Labeled coverage is collected
+        data = cov.get_data()
+        filenames = self.get_measured_filenames(data)
+        self.assertEqual(
+            sorted(data.measured_contexts()),
+            ['', 'doctest:HTML_TAG', 'test:HTML_TAG', 'test:RENDERERS'])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="doctest:HTML_TAG"),
+            [2])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="test:HTML_TAG"),
+            [2])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="test:RENDERERS"),
+            [2, 5, 8, 11])
+
+    def test_static_context(self):
+        self.make_plugin_capitalized_testnames('plugin_tests.py')
+        self.make_testsuite()
+
+        # Enable dynamic context plugin for coverage with named context
+        cov = coverage.Coverage(context='mytests')
+        cov.set_option("run:plugins", ['plugin_tests'])
+
+        # Run the tests
+        suite = self.run_testsuite(cov, 'testsuite')
+
+        # Static context prefix is preserved
+        data = cov.get_data()
+        filenames = self.get_measured_filenames(data)
+        self.assertEqual(
+            sorted(data.measured_contexts()),
+            ['mytests',
+             'mytests:doctest:HTML_TAG',
+             'mytests:test:HTML_TAG',
+             'mytests:test:RENDERERS'])
+
+    def test_plugin_with_test_function(self):
+        self.make_plugin_capitalized_testnames('plugin_tests.py')
+        self.make_testsuite()
+
+        # Enable both a plugin and test_function dynamic context
+        cov = coverage.Coverage()
+        cov.set_option("run:plugins", ['plugin_tests'])
+        cov.set_option("run:dynamic_context", "test_function")
+
+        # Run the tests
+        suite = self.run_testsuite(cov, 'testsuite')
+
+        # test_function takes precedence over plugins - only
+        # functions that are not labeled by test_function are
+        # labeled by plugin_tests.
+        data = cov.get_data()
+        filenames = self.get_measured_filenames(data)
+        self.assertEqual(
+            sorted(data.measured_contexts()),
+            ['', 'doctest:HTML_TAG', 'test_html_tag', 'test_renderers'])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="doctest:HTML_TAG"),
+            [2])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="test_html_tag"),
+            [2])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="test_renderers"),
+            [2, 5, 8, 11])
+
+    def test_multiple_plugins(self):
+        self.make_plugin_capitalized_testnames('plugin_tests.py')
+        self.make_plugin_track_render('plugin_renderers.py')
+        self.make_testsuite()
+
+        # Enable two plugins
+        cov = coverage.Coverage()
+        cov.set_option("run:plugins", ['plugin_renderers', 'plugin_tests'])
+
+        suite = self.run_testsuite(cov, 'testsuite')
+
+        # It is important to note, that line 11 (render_bold function) is never
+        # labeled as renderer:bold context, because it is only called from
+        # test_renderers function - so it already falls under test:RENDERERS
+        # context.
+        #
+        # render_paragraph and render_span (lines 5, 8) are directly called by
+        # testsuite.build_full_html, so they get labeled by renderers plugin.
+        data = cov.get_data()
+        filenames = self.get_measured_filenames(data)
+        self.assertEqual(
+            sorted(data.measured_contexts()),
+            ['',
+             'doctest:HTML_TAG',
+             'renderer:paragraph',
+             'renderer:span',
+             'test:HTML_TAG',
+             'test:RENDERERS'])
+
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="test:HTML_TAG"),
+            [2])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="test:RENDERERS"),
+            [2, 5, 8, 11])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="doctest:HTML_TAG"),
+            [2])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="renderer:paragraph"),
+            [2, 5])
+        self.assertEqual(
+            data.lines(filenames['rendering.py'], context="renderer:span"),
+            [2, 8])
+
+
+class DynamicContextPluginOtherTracersTest(CoverageTest):
+    """Tests of plugins that implement `dynamic_context`."""
+
+    def setUp(self):
+        super(DynamicContextPluginOtherTracersTest, self).setUp()
+        if env.C_TRACER:
+            self.skipTest("These tests are for tracers not implemented in C.")
+
+    def test_other_tracer_support(self):
+        self.make_file("context_plugin.py", """\
+            from coverage import CoveragePlugin
+
+            class Plugin(CoveragePlugin):
+                def dynamic_context(self, frame):
+                    return frame.f_code.co_name
+
+            def coverage_init(reg, options):
+                reg.add_dynamic_context(Plugin())
+            """)
+
+        cov = coverage.Coverage()
+        cov.set_option("run:plugins", ['context_plugin'])
+
+        with self.assertRaises(CoverageException):
+            cov.start()
+
+        cov.stop()