diff options
| -rw-r--r-- | CHANGES.rst | 23 | ||||
| -rw-r--r-- | coverage/collector.py | 69 | ||||
| -rw-r--r-- | coverage/config.py | 2 | ||||
| -rw-r--r-- | coverage/control.py | 20 | ||||
| -rw-r--r-- | coverage/ctracer/tracer.c | 5 | ||||
| -rw-r--r-- | coverage/ctracer/tracer.h | 3 | ||||
| -rw-r--r-- | coverage/sqldata.py | 2 | ||||
| -rw-r--r-- | doc/contexts.rst | 18 | ||||
| -rw-r--r-- | lab/find_class.py | 40 | ||||
| -rw-r--r-- | tests/test_context.py | 88 |
10 files changed, 220 insertions, 50 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 089fec7f..b449797b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -19,8 +19,27 @@ Unreleased - Context support: static contexts let you specify a label for a coverage run, which is recorded in the data, and retained when you combine files. See - :ref:`contexts` for more information. Currently, only static contexts are - supported, with no reporting features. + :ref:`contexts` for more information. + +- Dynamic contexts: specifying ``[run] dynamic_context = test_function`` in the + config file will record the test function name as a dynamic context during + execution. This is the core of "Who Tests What" (`issue 170`_). Things to + note: + + - There is no reporting support yet. Use SQLite to query the .coverage file + for information. Ideas are welcome about how reporting could be extended + to use this data. + + - There's a noticeable slow-down before any test is run. + + - Data files will now be roughly N times larger, where N is the number of + tests you have. Combining data files is therefore also N times slower. + + - No other values for ``dynamic_context`` are recognized yet. Let me know + what else would be useful. I'd like to use a pytest plugin to get better + information directly from pytest, for example. + +.. _issue 170: https://github.com/nedbat/coveragepy/issues/170 - Environment variable substitution in configuration files now supports two syntaxes for controlling the behavior of undefined variables: if ``VARNAME`` diff --git a/coverage/collector.py b/coverage/collector.py index e0144979..4e7058a0 100644 --- a/coverage/collector.py +++ b/coverage/collector.py @@ -34,14 +34,6 @@ except ImportError: CTracer = None -def should_start_context(frame): - """Who-Tests-What hack: Determine whether this frame begins a new who-context.""" - fn_name = frame.f_code.co_name - if fn_name.startswith("test"): - return fn_name - return None - - class Collector(object): """Collects trace data. @@ -66,7 +58,10 @@ class Collector(object): # The concurrency settings we support here. SUPPORTED_CONCURRENCIES = set(["greenlet", "eventlet", "gevent", "thread"]) - def __init__(self, should_trace, check_include, timid, branch, warn, concurrency): + def __init__( + self, should_trace, check_include, should_start_context, + timid, branch, warn, concurrency, + ): """Create a collector. `should_trace` is a function, taking a file name and a frame, and @@ -75,6 +70,11 @@ class Collector(object): `check_include` is a function taking a file name and a frame. It returns a boolean: True if the file should be traced, False if not. + `should_start_context` is a function taking a frame, and returning a + string. If the frame should be the start of a new context, the string + is the new context. If the frame should not be the start of a new + context, return None. + If `timid` is true, then a slower simpler trace function will be used. This is important for some environments where manipulation of tracing functions make the faster more sophisticated trace function not @@ -96,6 +96,7 @@ class Collector(object): """ self.should_trace = should_trace self.check_include = check_include + self.should_start_context = should_start_context self.warn = warn self.branch = branch self.threading = None @@ -139,10 +140,6 @@ class Collector(object): ) ) - # Who-Tests-What is just a hack at the moment, so turn it on with an - # environment variable. - self.wtw = int(os.getenv('COVERAGE_WTW', 0)) - self.reset() if timid: @@ -175,7 +172,11 @@ class Collector(object): def _clear_data(self): """Clear out existing data, but stay ready for more collection.""" - self.data.clear() + # We used to used self.data.clear(), but that would remove filename + # keys and data values that were still in use higher up the stack + # when we are called as part of switch_context. + for d in self.data.values(): + d.clear() for tracer in self.tracers: tracer.reset_activity() @@ -187,10 +188,6 @@ class Collector(object): # pairs as keys (if branch coverage). self.data = {} - # A dict mapping contexts to data dictionaries. - self.contexts = {} - self.contexts[None] = self.data - # A dictionary mapping file names to file tracer plugin names that will # handle them. self.file_tracers = {} @@ -252,11 +249,13 @@ class Collector(object): tracer.threading = self.threading if hasattr(tracer, 'check_include'): tracer.check_include = self.check_include - if self.wtw: - if hasattr(tracer, 'should_start_context'): - tracer.should_start_context = should_start_context - if hasattr(tracer, 'switch_context'): - tracer.switch_context = self.switch_context + if hasattr(tracer, 'should_start_context'): + tracer.should_start_context = self.should_start_context + tracer.switch_context = self.switch_context + elif self.should_start_context: + raise CoverageException( + "Can't support dynamic contexts with {}".format(self.tracer_name()) + ) fn = tracer.start() self.tracers.append(tracer) @@ -372,12 +371,15 @@ class Collector(object): return any(tracer.activity() for tracer in self.tracers) def switch_context(self, new_context): - """Who-Tests-What hack: switch to a new who-context.""" - # Make a new data dict, or find the existing one, and switch all the - # tracers to use it. - data = self.contexts.setdefault(new_context, {}) - for tracer in self.tracers: - tracer.data = data + """Switch to a new dynamic context.""" + self.flush_data() + if self.static_context: + context = self.static_context + if new_context: + context += ":" + new_context + else: + context = new_context + self.covdata.set_context(context) def cached_abs_file(self, filename): """A locally cached version of `abs_file`.""" @@ -415,7 +417,7 @@ class Collector(object): else: raise runtime_err # pylint: disable=raising-bad-type - return dict((self.cached_abs_file(k), v) for k, v in items) + return dict((self.cached_abs_file(k), v) for k, v in items if v) if self.branch: self.covdata.add_arcs(abs_file_dict(self.data)) @@ -423,12 +425,5 @@ class Collector(object): self.covdata.add_lines(abs_file_dict(self.data)) self.covdata.add_file_tracers(abs_file_dict(self.file_tracers)) - if self.wtw: - # Just a hack, so just hack it. - import pprint - out_file = "coverage_wtw_{:06}.py".format(os.getpid()) - with open(out_file, "w") as wtw_out: - pprint.pprint(self.contexts, wtw_out) - self._clear_data() return True diff --git a/coverage/config.py b/coverage/config.py index 9a11323d..2a281875 100644 --- a/coverage/config.py +++ b/coverage/config.py @@ -180,6 +180,7 @@ class CoverageConfig(object): self.data_file = ".coverage" self.debug = [] self.disable_warnings = [] + self.dynamic_context = None self.note = None self.parallel = False self.plugins = [] @@ -324,6 +325,7 @@ class CoverageConfig(object): ('data_file', 'run:data_file'), ('debug', 'run:debug', 'list'), ('disable_warnings', 'run:disable_warnings', 'list'), + ('dynamic_context', 'run:dynamic_context'), ('note', 'run:note'), ('parallel', 'run:parallel', 'boolean'), ('plugins', 'run:plugins', 'list'), diff --git a/coverage/control.py b/coverage/control.py index 678a7b3b..4f2afda0 100644 --- a/coverage/control.py +++ b/coverage/control.py @@ -347,9 +347,19 @@ class Coverage(object): # it for the main process. self.config.parallel = True + if self.config.dynamic_context is None: + should_start_context = None + elif self.config.dynamic_context == "test_function": + should_start_context = should_start_context_test_function + else: + raise CoverageException( + "Don't understand dynamic_context setting: {!r}".format(self.config.dynamic_context) + ) + self._collector = Collector( should_trace=self._should_trace, check_include=self._check_include_omit_etc, + should_start_context=should_start_context, timid=self.config.timid, branch=self.config.branch, warn=self._warn, @@ -886,6 +896,16 @@ if int(os.environ.get("COVERAGE_DEBUG_CALLS", 0)): # pragma: debugg Coverage = decorate_methods(show_calls(show_args=True), butnot=['get_data'])(Coverage) +def should_start_context_test_function(frame): + """Who-Tests-What hack: Determine whether this frame begins a new who-context.""" + with open("/tmp/ssc.txt", "a") as f: + f.write("hello\n") + fn_name = frame.f_code.co_name + if fn_name.startswith("test"): + return fn_name + return None + + def process_startup(): """Call this at Python start-up to perhaps measure coverage. diff --git a/coverage/ctracer/tracer.c b/coverage/ctracer/tracer.c index 01f8b19b..7d639112 100644 --- a/coverage/ctracer/tracer.c +++ b/coverage/ctracer/tracer.c @@ -341,7 +341,6 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) CFileDisposition * pdisp = NULL; STATS( self->stats.calls++; ) - self->activity = TRUE; /* Grow the stack. */ if (CTracer_set_pdata_stack(self) < 0) { @@ -353,7 +352,7 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) self->pcur_entry = &self->pdata_stack->stack[self->pdata_stack->depth]; /* See if this frame begins a new context. */ - if (self->should_start_context && self->context == Py_None) { + if (self->should_start_context != Py_None && self->context == Py_None) { PyObject * context; /* We're looking for our context, ask should_start_context if this is the start. */ STATS( self->stats.start_context_calls++; ) @@ -866,6 +865,8 @@ CTracer_trace(CTracer *self, PyFrameObject *frame, int what, PyObject *arg_unuse goto error; } + self->activity = TRUE; + switch (what) { case PyTrace_CALL: if (CTracer_handle_call(self, frame) < 0) { diff --git a/coverage/ctracer/tracer.h b/coverage/ctracer/tracer.h index 61c01b41..a83742dd 100644 --- a/coverage/ctracer/tracer.h +++ b/coverage/ctracer/tracer.h @@ -27,7 +27,6 @@ typedef struct CTracer { PyObject * trace_arcs; PyObject * should_start_context; PyObject * switch_context; - PyObject * context; /* Has the tracer been started? */ BOOL started; @@ -35,6 +34,8 @@ typedef struct CTracer { BOOL tracing_arcs; /* Have we had any activity? */ BOOL activity; + /* The current dynamic context. */ + PyObject * context; /* The data stack is a stack of dictionaries. Each dictionary collects diff --git a/coverage/sqldata.py b/coverage/sqldata.py index fb2279c9..738fccef 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -199,6 +199,8 @@ class CoverageSqliteData(SimpleReprMixin): def set_context(self, context): """Set the current context for future `add_lines` etc.""" + if self._debug and self._debug.should('dataop'): + self._debug.write("Setting context: %r" % (context,)) self._start_using() context = context or "" with self._connect() as con: diff --git a/doc/contexts.rst b/doc/contexts.rst index c1d4a173..1f1ce763 100644 --- a/doc/contexts.rst +++ b/doc/contexts.rst @@ -16,9 +16,8 @@ in which it was run. This can provide more information to help you understand the behavior of your tests. There are two kinds of context: static and dynamic. Static contexts are fixed -for an entire run, and are set explicitly with an option. - -Dynamic contexts are coming soon. +for an entire run, and are set explicitly with an option. Dynamic contexts +change over the course of a single run. Static contexts @@ -39,7 +38,18 @@ A static context is specified with the ``--context=CONTEXT`` option to Dynamic contexts ---------------- -Not implemented yet. +Dynamic contexts are found during execution. There is currently support for +one kind: test function names. Set the ``dynamic_context`` option to +``test_function`` in the configuration file:: + + [run] + dynamic_context = test_function + +Each test function you run will be considered a separate dynamic context, and +coverage data will be segregated for each. A test function is any function +whose names starts with "test". + +Ideas are welcome for other dynamic contexts that would be useful. Context reporting diff --git a/lab/find_class.py b/lab/find_class.py new file mode 100644 index 00000000..d8dac0b5 --- /dev/null +++ b/lab/find_class.py @@ -0,0 +1,40 @@ +class Parent(object): + def meth(self): + print("METH") + +class Child(Parent): + pass + +def trace(frame, event, args): + # Thanks to Aleksi Torhamo for code and idea. + co = frame.f_code + fname = co.co_name + if not co.co_varnames: + return + locs = frame.f_locals + first_arg = co.co_varnames[0] + if co.co_argcount: + self = locs[first_arg] + elif co.co_flags & 0x04: # *args syntax + self = locs[first_arg][0] + else: + return + + func = getattr(self, fname).__func__ + if hasattr(func, '__qualname__'): + qname = func.__qualname__ + else: + for cls in self.__class__.__mro__: + f = cls.__dict__.get(fname, None) + if f is None: + continue + if f is func: + qname = cls.__name__ + "." + fname + break + print("{}: {}.{} {}".format(event, self, fname, qname)) + return trace + +import sys +sys.settrace(trace) + +Child().meth() diff --git a/tests/test_context.py b/tests/test_context.py index a6be922d..4339d336 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -6,16 +6,18 @@ import os.path import coverage +from coverage import env from coverage.data import CoverageData +from coverage.misc import CoverageException from tests.coveragetest import CoverageTest -class GlobalContextTest(CoverageTest): - """Tests of the global context.""" +class StaticContextTest(CoverageTest): + """Tests of the static context.""" def setUp(self): - super(GlobalContextTest, self).setUp() + super(StaticContextTest, self).setUp() self.skip_unless_data_storage_is("sql") def test_no_context(self): @@ -25,7 +27,7 @@ class GlobalContextTest(CoverageTest): data = cov.get_data() self.assertCountEqual(data.measured_contexts(), [""]) - def test_global_context(self): + def test_static_context(self): self.make_file("main.py", "a = 1") cov = coverage.Coverage(context="gooey") self.start_import_stop(cov, "main") @@ -102,3 +104,81 @@ class GlobalContextTest(CoverageTest): self.assertEqual(combined.arcs(fred, context='blue'), []) self.assertEqual(combined.arcs(fblue, context='red'), []) self.assertEqual(combined.arcs(fblue, context='blue'), self.ARCS) + + +class DynamicContextTest(CoverageTest): + """Tests of dynamically changing contexts.""" + + def setUp(self): + super(DynamicContextTest, self).setUp() + self.skip_unless_data_storage_is("sql") + if not env.C_TRACER: + self.skipTest("Only the C tracer supports dynamic contexts") + + SOURCE = """\ + def helper(lineno): + x = 2 + + def test_one(): + a = 5 + helper(6) + + def test_two(): + a = 9 + b = 10 + if a > 11: + b = 12 + assert a == (13-4) + assert b == (14-4) + helper(15) + + test_one() + x = 18 + helper(19) + test_two() + """ + + OUTER_LINES = [1, 4, 8, 17, 18, 19, 2, 20] + TEST_ONE_LINES = [5, 6, 2] + TEST_TWO_LINES = [9, 10, 11, 13, 14, 15, 2] + + def test_dynamic_alone(self): + self.make_file("two_tests.py", self.SOURCE) + cov = coverage.Coverage(source=["."]) + cov.set_option("run:dynamic_context", "test_function") + self.start_import_stop(cov, "two_tests") + data = cov.get_data() + + fname = os.path.abspath("two_tests.py") + self.assertCountEqual(data.measured_contexts(), ["", "test_one", "test_two"]) + self.assertCountEqual(data.lines(fname, ""), self.OUTER_LINES) + self.assertCountEqual(data.lines(fname, "test_one"), self.TEST_ONE_LINES) + self.assertCountEqual(data.lines(fname, "test_two"), self.TEST_TWO_LINES) + + def test_static_and_dynamic(self): + self.make_file("two_tests.py", self.SOURCE) + cov = coverage.Coverage(context="stat", source=["."]) + cov.set_option("run:dynamic_context", "test_function") + self.start_import_stop(cov, "two_tests") + data = cov.get_data() + + fname = os.path.abspath("two_tests.py") + self.assertCountEqual(data.measured_contexts(), ["stat", "stat:test_one", "stat:test_two"]) + self.assertCountEqual(data.lines(fname, "stat"), self.OUTER_LINES) + self.assertCountEqual(data.lines(fname, "stat:test_one"), self.TEST_ONE_LINES) + self.assertCountEqual(data.lines(fname, "stat:test_two"), self.TEST_TWO_LINES) + + +class DynamicContextWithPythonTracerTest(CoverageTest): + """The Python tracer doesn't do dynamic contexts at all.""" + + run_in_temp_dir = False + + def test_python_tracer_fails_properly(self): + if env.C_TRACER: + self.skipTest("This test is specifically about the Python tracer.") + cov = coverage.Coverage() + cov.set_option("run:dynamic_context", "test_function") + msg = r"Can't support dynamic contexts with PyTracer" + with self.assertRaisesRegex(CoverageException, msg): + cov.start() |