From 181f5a78fdbdb7d6f90a478482512297f3a0f845 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 9 Sep 2018 09:38:27 -0400 Subject: Super-simple contexts added to the schema --- coverage/sqldata.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'coverage') diff --git a/coverage/sqldata.py b/coverage/sqldata.py index 91508586..0c05ae1f 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -22,7 +22,11 @@ from coverage.files import PathAliases from coverage.misc import CoverageException, file_be_gone -SCHEMA_VERSION = 1 +# Schema versions: +# 1: Released in 5.0a2 +# 2: Added contexts + +SCHEMA_VERSION = 2 SCHEMA = """ create table coverage_schema ( @@ -40,17 +44,25 @@ create table file ( unique(path) ); +create table context ( + id integer primary key, + context text, + unique(context) +); + create table line ( file_id integer, + context_id integer, lineno integer, - unique(file_id, lineno) + unique(file_id, context_id, lineno) ); create table arc ( file_id integer, + context_id integer, fromno integer, tono integer, - unique(file_id, fromno, tono) + unique(file_id, context_id, fromno, tono) ); create table tracer ( @@ -78,6 +90,8 @@ class CoverageSqliteData(SimpleRepr): self._has_lines = False self._has_arcs = False + self._context_id = 0 + def _choose_filename(self): self.filename = self._basename suffix = filename_suffix(self._suffix) @@ -181,9 +195,9 @@ class CoverageSqliteData(SimpleRepr): with self._connect() as con: for filename, linenos in iitems(line_data): file_id = self._file_id(filename, add=True) - data = [(file_id, lineno) for lineno in linenos] + data = [(file_id, self._context_id, lineno) for lineno in linenos] con.executemany( - "insert or ignore into line (file_id, lineno) values (?, ?)", + "insert or ignore into line (file_id, context_id, lineno) values (?, ?, ?)", data, ) @@ -204,9 +218,9 @@ class CoverageSqliteData(SimpleRepr): with self._connect() as con: for filename, arcs in iitems(arc_data): file_id = self._file_id(filename, add=True) - data = [(file_id, fromno, tono) for fromno, tono in arcs] + data = [(file_id, self._context_id, fromno, tono) for fromno, tono in arcs] con.executemany( - "insert or ignore into arc (file_id, fromno, tono) values (?, ?, ?)", + "insert or ignore into arc (file_id, context_id, fromno, tono) values (?, ?, ?, ?)", data, ) -- cgit v1.2.1 From 5aa95d1edec75c4f30458773894c7f47c1af0edc Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 9 Sep 2018 10:58:38 -0400 Subject: Plumb through context= setting --- coverage/cmdline.py | 7 +++++++ coverage/config.py | 2 ++ coverage/control.py | 6 ++++-- 3 files changed, 13 insertions(+), 2 deletions(-) (limited to 'coverage') diff --git a/coverage/cmdline.py b/coverage/cmdline.py index 23d2aec3..99b155b2 100644 --- a/coverage/cmdline.py +++ b/coverage/cmdline.py @@ -43,6 +43,10 @@ class Opts(object): "Valid values are: %s." ) % ", ".join(CONCURRENCY_CHOICES), ) + context = optparse.make_option( + '', '--context', action='store', metavar="LABEL", + help="The context label to record for this coverage run", + ) debug = optparse.make_option( '', '--debug', action='store', metavar="OPTS", help="Debug options, separated by commas. [env: COVERAGE_DEBUG]", @@ -160,6 +164,7 @@ class CoverageOptionParser(optparse.OptionParser, object): append=None, branch=None, concurrency=None, + context=None, debug=None, directory=None, fail_under=None, @@ -358,6 +363,7 @@ CMDS = { Opts.append, Opts.branch, Opts.concurrency, + Opts.context, Opts.include, Opts.module, Opts.omit, @@ -482,6 +488,7 @@ class CoverageScript(object): debug=debug, concurrency=options.concurrency, check_preimported=True, + context=options.context, ) if options.action == "debug": diff --git a/coverage/config.py b/coverage/config.py index 69c929b4..9a11323d 100644 --- a/coverage/config.py +++ b/coverage/config.py @@ -175,6 +175,7 @@ class CoverageConfig(object): # Defaults for [run] self.branch = False self.concurrency = None + self.context = None self.cover_pylib = False self.data_file = ".coverage" self.debug = [] @@ -318,6 +319,7 @@ class CoverageConfig(object): # [run] ('branch', 'run:branch', 'boolean'), ('concurrency', 'run:concurrency', 'list'), + ('context', 'run:context'), ('cover_pylib', 'run:cover_pylib', 'boolean'), ('data_file', 'run:data_file'), ('debug', 'run:debug', 'list'), diff --git a/coverage/control.py b/coverage/control.py index 5d42af77..cdbd721f 100644 --- a/coverage/control.py +++ b/coverage/control.py @@ -57,7 +57,7 @@ class Coverage(object): self, data_file=None, data_suffix=None, cover_pylib=None, auto_data=False, timid=None, branch=None, config_file=True, source=None, omit=None, include=None, debug=None, - concurrency=None, check_preimported=False, + concurrency=None, check_preimported=False, context=None, ): """ `data_file` is the base name of the data file to use, defaulting to @@ -116,6 +116,8 @@ class Coverage(object): by coverage. Importing measured files before coverage is started can mean that code is missed. + `context` is a string to use as the context label for collected data. + .. versionadded:: 4.0 The `concurrency` parameter. @@ -133,7 +135,7 @@ class Coverage(object): branch=branch, parallel=bool_or_none(data_suffix), source=source, run_omit=omit, run_include=include, debug=debug, report_omit=omit, report_include=include, - concurrency=concurrency, + concurrency=concurrency, context=context, ) # This is injectable by tests. -- cgit v1.2.1 From 3839a376d03f22fad41b869be680ba496147b281 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 9 Sep 2018 13:19:40 -0400 Subject: Collector has a CoverageData --- coverage/collector.py | 18 +++++++++++++----- coverage/control.py | 6 +++++- 2 files changed, 18 insertions(+), 6 deletions(-) (limited to 'coverage') diff --git a/coverage/collector.py b/coverage/collector.py index fa3eaaa4..db8373a1 100644 --- a/coverage/collector.py +++ b/coverage/collector.py @@ -99,6 +99,7 @@ class Collector(object): self.warn = warn self.branch = branch self.threading = None + self.covdata = None self.origin = short_stack() @@ -160,6 +161,10 @@ class Collector(object): def __repr__(self): return "" % (id(self), self.tracer_name()) + def use_data(self, covdata): + """Use `covdata` for recording data.""" + self.covdata = covdata + def tracer_name(self): """Return the class name of the tracer we're using.""" return self._trace_class.__name__ @@ -378,8 +383,11 @@ class Collector(object): except KeyError: return self.abs_file_cache.setdefault(key, abs_file(filename)) - def save_data(self, covdata): - """Save the collected data to a `CoverageData`. + def flush_data(self): + """Save the collected data to our associated `CoverageData`. + + Data may have also been saved along the way. This forces the + last of the data to be saved. Returns True if there was data to save, False if not. """ @@ -406,10 +414,10 @@ class Collector(object): return dict((self.cached_abs_file(k), v) for k, v in items) if self.branch: - covdata.add_arcs(abs_file_dict(self.data)) + self.covdata.add_arcs(abs_file_dict(self.data)) else: - covdata.add_lines(abs_file_dict(self.data)) - covdata.add_file_tracers(abs_file_dict(self.file_tracers)) + self.covdata.add_lines(abs_file_dict(self.data)) + self.covdata.add_file_tracers(abs_file_dict(self.file_tracers)) if self.wtw: # Just a hack, so just hack it. diff --git a/coverage/control.py b/coverage/control.py index cdbd721f..ca0843d7 100644 --- a/coverage/control.py +++ b/coverage/control.py @@ -335,6 +335,7 @@ class Coverage(object): def _init_for_start(self): """Initialization for start()""" + # Construct the collector. concurrency = self.config.concurrency or [] if "multiprocessing" in concurrency: if not patch_multiprocessing: @@ -400,6 +401,9 @@ class Coverage(object): debug=self._debug, ) + if self._collector is not None: + self._collector.use_data(self._data) + def start(self): """Start measuring code coverage. @@ -564,7 +568,7 @@ class Coverage(object): self._init_data(suffix=None) self._post_init() - if self._collector and self._collector.save_data(self._data): + if self._collector and self._collector.flush_data(): self._post_save_work() return self._data -- cgit v1.2.1 From d5d3427c92b78324beaba4babb281ac96eb1ebc1 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 10 Sep 2018 15:19:21 -0400 Subject: SqlData can set_context --- coverage/sqldata.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'coverage') diff --git a/coverage/sqldata.py b/coverage/sqldata.py index 0c05ae1f..641e8ae1 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -178,6 +178,20 @@ class CoverageSqliteData(SimpleRepr): self._file_map[filename] = cur.lastrowid return self._file_map.get(filename) + def set_context(self, context): + """Get the context id for `context`.""" + self._start_using() + if not context: + self._context_id = 0 + else: + with self._connect() as con: + row = con.execute("select id from context where context = ?", (context,)).fetchone() + if row is not None: + self._context_id = row[0] + else: + cur = con.execute("insert into context (context) values (?)", (context,)) + self._context_id = cur.lastrowid + def add_lines(self, line_data): """Add measured line data. -- cgit v1.2.1 From 8c05c99a4782e67c75e361f3fcb06ae32559e5f1 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 12 Sep 2018 08:39:37 -0400 Subject: Set the context in the data --- coverage/collector.py | 6 +++++- coverage/control.py | 5 ++--- coverage/sqldata.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'coverage') diff --git a/coverage/collector.py b/coverage/collector.py index db8373a1..e0144979 100644 --- a/coverage/collector.py +++ b/coverage/collector.py @@ -101,6 +101,8 @@ class Collector(object): self.threading = None self.covdata = None + self.static_context = None + self.origin = short_stack() self.concur_id_func = None @@ -161,9 +163,11 @@ class Collector(object): def __repr__(self): return "" % (id(self), self.tracer_name()) - def use_data(self, covdata): + def use_data(self, covdata, context): """Use `covdata` for recording data.""" self.covdata = covdata + self.static_context = context + self.covdata.set_context(self.static_context) def tracer_name(self): """Return the class name of the tracer we're using.""" diff --git a/coverage/control.py b/coverage/control.py index ca0843d7..0918a34e 100644 --- a/coverage/control.py +++ b/coverage/control.py @@ -366,6 +366,8 @@ class Coverage(object): self._init_data(suffix) + self._collector.use_data(self._data, self.config.context) + # Early warning if we aren't going to be able to support plugins. if self._plugins.file_tracers and not self._collector.supports_plugins: self._warn( @@ -401,9 +403,6 @@ class Coverage(object): debug=self._debug, ) - if self._collector is not None: - self._collector.use_data(self._data) - def start(self): """Start measuring code coverage. diff --git a/coverage/sqldata.py b/coverage/sqldata.py index 641e8ae1..6dde9c2e 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -111,7 +111,7 @@ class CoverageSqliteData(SimpleRepr): self._db = Sqlite(self.filename, self._debug) with self._db: for stmt in SCHEMA.split(';'): - stmt = stmt.strip() + stmt = " ".join(stmt.strip().split()) if stmt: self._db.execute(stmt) self._db.execute("insert into coverage_schema (version) values (?)", (SCHEMA_VERSION,)) -- cgit v1.2.1 From 2f1b8cfcfe184a8fd6f3f2f789530bddb233dda8 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 14 Sep 2018 07:05:51 -0400 Subject: Change measured_files to a set --- coverage/control.py | 10 +++------- coverage/data.py | 4 ++-- coverage/sqldata.py | 6 +++--- 3 files changed, 8 insertions(+), 12 deletions(-) (limited to 'coverage') diff --git a/coverage/control.py b/coverage/control.py index 0918a34e..f7d97cf6 100644 --- a/coverage/control.py +++ b/coverage/control.py @@ -683,15 +683,11 @@ class Coverage(object): if not morfs: morfs = self._data.measured_files() - # Be sure we have a list. - if not isinstance(morfs, (list, tuple)): + # Be sure we have a collection. + if not isinstance(morfs, (list, tuple, set)): morfs = [morfs] - file_reporters = [] - for morf in morfs: - file_reporter = self._get_file_reporter(morf) - file_reporters.append(file_reporter) - + file_reporters = [self._get_file_reporter(morf) for morf in morfs] return file_reporters def report( diff --git a/coverage/data.py b/coverage/data.py index f03e90ca..3250196d 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -252,8 +252,8 @@ class CoverageJsonData(object): return self._runs def measured_files(self): - """A list of all files that had been measured.""" - return list(self._arcs or self._lines or {}) + """A set of all files that had been measured.""" + return set(self._arcs or self._lines or {}) def __nonzero__(self): return bool(self._lines or self._arcs) diff --git a/coverage/sqldata.py b/coverage/sqldata.py index 6dde9c2e..b9488557 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -304,7 +304,7 @@ class CoverageSqliteData(SimpleRepr): aliases = aliases or PathAliases() # See what we had already measured, for accurate conflict reporting. - this_measured = set(self.measured_files()) + this_measured = self.measured_files() # lines if other_data._has_lines: @@ -381,8 +381,8 @@ class CoverageSqliteData(SimpleRepr): return bool(self._has_arcs) def measured_files(self): - """A list of all files that had been measured.""" - return list(self._file_map) + """A set of all files that had been measured.""" + return set(self._file_map) def file_tracer(self, filename): """Get the plugin name of the file tracer for a file. -- cgit v1.2.1 From d2f77ab2ffc308e616af0207546ee1bef1cb8c75 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 15 Sep 2018 08:07:26 -0400 Subject: measured_contexts() and two simple tests of the global context --- coverage/data.py | 5 +++++ coverage/sqldata.py | 27 ++++++++++++++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) (limited to 'coverage') diff --git a/coverage/data.py b/coverage/data.py index 3250196d..3a2432b3 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -445,6 +445,11 @@ class CoverageJsonData(object): self._validate() + def set_context(self, context): + """Set the context. Not implemented for JSON storage.""" + if context: + raise CoverageException("JSON storage doesn't support contexts") + def write(self): """Write the collected coverage data to a file. diff --git a/coverage/sqldata.py b/coverage/sqldata.py index b9488557..45c1570c 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -179,18 +179,16 @@ class CoverageSqliteData(SimpleRepr): return self._file_map.get(filename) def set_context(self, context): - """Get the context id for `context`.""" + """Set the current context for future `add_lines` etc.""" self._start_using() - if not context: - self._context_id = 0 - else: - with self._connect() as con: - row = con.execute("select id from context where context = ?", (context,)).fetchone() - if row is not None: - self._context_id = row[0] - else: - cur = con.execute("insert into context (context) values (?)", (context,)) - self._context_id = cur.lastrowid + context = context or "" + with self._connect() as con: + row = con.execute("select id from context where context = ?", (context,)).fetchone() + if row is not None: + self._context_id = row[0] + else: + cur = con.execute("insert into context (context) values (?)", (context,)) + self._context_id = cur.lastrowid def add_lines(self, line_data): """Add measured line data. @@ -384,6 +382,13 @@ class CoverageSqliteData(SimpleRepr): """A set of all files that had been measured.""" return set(self._file_map) + def measured_contexts(self): + """A set of all contexts that have been measured.""" + self._start_using() + with self._connect() as con: + contexts = set(row[0] for row in con.execute("select distinct(context) from context")) + return contexts + def file_tracer(self, filename): """Get the plugin name of the file tracer for a file. -- cgit v1.2.1 From 5ff763737475f8fa1a587f6903de1329b41090ae Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 18 Sep 2018 07:49:54 -0400 Subject: Combining contexts works --- coverage/sqldata.py | 74 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 22 deletions(-) (limited to 'coverage') diff --git a/coverage/sqldata.py b/coverage/sqldata.py index 45c1570c..224573be 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -12,6 +12,7 @@ # TODO: run_info import glob +import itertools import os import sqlite3 @@ -90,7 +91,7 @@ class CoverageSqliteData(SimpleRepr): self._has_lines = False self._has_arcs = False - self._context_id = 0 + self._current_context_id = None def _choose_filename(self): self.filename = self._basename @@ -104,6 +105,7 @@ class CoverageSqliteData(SimpleRepr): self._db = None self._file_map = {} self._have_used = False + self._current_context_id = None def _create_db(self): if self._debug and self._debug.should('dataio'): @@ -178,6 +180,17 @@ class CoverageSqliteData(SimpleRepr): self._file_map[filename] = cur.lastrowid return self._file_map.get(filename) + def _context_id(self, context): + """Get the id for a context.""" + assert context is not None + self._start_using() + with self._connect() as con: + row = con.execute("select id from context where context = ?", (context,)).fetchone() + if row is not None: + return row[0] + else: + return None + def set_context(self, context): """Set the current context for future `add_lines` etc.""" self._start_using() @@ -185,10 +198,10 @@ class CoverageSqliteData(SimpleRepr): with self._connect() as con: row = con.execute("select id from context where context = ?", (context,)).fetchone() if row is not None: - self._context_id = row[0] + self._current_context_id = row[0] else: cur = con.execute("insert into context (context) values (?)", (context,)) - self._context_id = cur.lastrowid + self._current_context_id = cur.lastrowid def add_lines(self, line_data): """Add measured line data. @@ -204,10 +217,12 @@ class CoverageSqliteData(SimpleRepr): )) self._start_using() self._choose_lines_or_arcs(lines=True) + if self._current_context_id is None: + self.set_context("") with self._connect() as con: for filename, linenos in iitems(line_data): file_id = self._file_id(filename, add=True) - data = [(file_id, self._context_id, lineno) for lineno in linenos] + data = [(file_id, self._current_context_id, lineno) for lineno in linenos] con.executemany( "insert or ignore into line (file_id, context_id, lineno) values (?, ?, ?)", data, @@ -227,10 +242,12 @@ class CoverageSqliteData(SimpleRepr): )) self._start_using() self._choose_lines_or_arcs(arcs=True) + if self._current_context_id is None: + self.set_context("") with self._connect() as con: for filename, arcs in iitems(arc_data): file_id = self._file_id(filename, add=True) - data = [(file_id, self._context_id, fromno, tono) for fromno, tono in arcs] + data = [(file_id, self._current_context_id, fromno, tono) for fromno, tono in arcs] con.executemany( "insert or ignore into arc (file_id, context_id, fromno, tono) values (?, ?, ?, ?)", data, @@ -306,19 +323,23 @@ class CoverageSqliteData(SimpleRepr): # lines if other_data._has_lines: - for filename in other_data.measured_files(): - lines = set(other_data.lines(filename)) - filename = aliases.map(filename) - lines.update(self.lines(filename) or ()) - self.add_lines({filename: lines}) + for context in other_data.measured_contexts(): + self.set_context(context) + for filename in other_data.measured_files(): + lines = set(other_data.lines(filename, context=context)) + filename = aliases.map(filename) + lines.update(self.lines(filename, context=context) or ()) + self.add_lines({filename: lines}) # arcs if other_data._has_arcs: - for filename in other_data.measured_files(): - arcs = set(other_data.arcs(filename)) - filename = aliases.map(filename) - arcs.update(self.arcs(filename) or ()) - self.add_arcs({filename: arcs}) + for context in other_data.measured_contexts(): + self.set_context(context) + for filename in other_data.measured_files(): + arcs = set(other_data.arcs(filename, context=context)) + filename = aliases.map(filename) + arcs.update(self.arcs(filename, context=context) or ()) + self.add_arcs({filename: arcs}) # file_tracers for filename in other_data.measured_files(): @@ -407,12 +428,11 @@ class CoverageSqliteData(SimpleRepr): return row[0] or "" return "" # File was measured, but no tracer associated. - def lines(self, filename): + def lines(self, filename, context=None): self._start_using() if self.has_arcs(): - arcs = self.arcs(filename) + arcs = self.arcs(filename, context=context) if arcs is not None: - import itertools all_lines = itertools.chain.from_iterable(arcs) return list(set(l for l in all_lines if l > 0)) @@ -421,18 +441,28 @@ class CoverageSqliteData(SimpleRepr): if file_id is None: return None else: - linenos = con.execute("select lineno from line where file_id = ?", (file_id,)) + query = "select lineno from line where file_id = ?" + data = [file_id] + if context is not None: + query += " and context_id = ?" + data += [self._context_id(context)] + linenos = con.execute(query, data) return [lineno for lineno, in linenos] - def arcs(self, filename): + def arcs(self, filename, context=None): self._start_using() with self._connect() as con: file_id = self._file_id(filename) if file_id is None: return None else: - arcs = con.execute("select fromno, tono from arc where file_id = ?", (file_id,)) - return [pair for pair in arcs] + query = "select fromno, tono from arc where file_id = ?" + data = [file_id] + if context is not None: + query += " and context_id = ?" + data += [self._context_id(context)] + arcs = con.execute(query, data) + return list(arcs) def run_infos(self): return [] # TODO -- cgit v1.2.1 From edc25b9a723272f869c598e929d72e5db341ba0d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 20 Sep 2018 19:45:17 -0400 Subject: More debugging. --- coverage/data.py | 2 ++ coverage/sqldata.py | 10 ++++++++++ 2 files changed, 12 insertions(+) (limited to 'coverage') diff --git a/coverage/data.py b/coverage/data.py index 3a2432b3..e6d56d84 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -727,6 +727,8 @@ def combine_parallel_data(data, aliases=None, data_paths=None, strict=False): files_combined = 0 for f in files_to_combine: + if data._debug and data._debug.should('dataio'): + data._debug.write("Combining data file %r" % (f,)) try: new_data = CoverageData(f, debug=data._debug) new_data.read() diff --git a/coverage/sqldata.py b/coverage/sqldata.py index 224573be..e644ec16 100644 --- a/coverage/sqldata.py +++ b/coverage/sqldata.py @@ -167,6 +167,12 @@ class CoverageSqliteData(SimpleRepr): __bool__ = __nonzero__ + def dump(self): # pragma: debugging + """Write a dump of the database.""" + if self._debug: + with self._connect() as con: + self._debug.write(con.dump()) + def _file_id(self, filename, add=False): """Get the file id for `filename`. @@ -519,3 +525,7 @@ class Sqlite(SimpleRepr): if self.debug: self.debug.write("Executing many {!r} with {} rows".format(sql, len(data))) return self.con.executemany(sql, data) + + def dump(self): # pragma: debugging + """Return a multi-line string, the dump of the database.""" + return "\n".join(self.con.iterdump()) -- cgit v1.2.1