diff options
Diffstat (limited to 'coverage')
46 files changed, 3371 insertions, 1760 deletions
diff --git a/coverage/__init__.py b/coverage/__init__.py index 0aa1d45c..d132e4a7 100644 --- a/coverage/__init__.py +++ b/coverage/__init__.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Code coverage measurement for Python. Ned Batchelder @@ -5,78 +8,16 @@ http://nedbatchelder.com/code/coverage """ -from coverage.version import __version__, __url__ +from coverage.version import __version__, __url__, version_info from coverage.control import Coverage, process_startup from coverage.data import CoverageData -from coverage.cmdline import main, CoverageScript from coverage.misc import CoverageException -from coverage.plugin import CoveragePlugin +from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # Backward compatibility. coverage = Coverage -# Module-level functions. The original API to this module was based on -# functions defined directly in the module, with a singleton of the Coverage() -# class. That design hampered programmability, so the current API uses -# explicitly-created Coverage objects. But for backward compatibility, here we -# define the top-level functions to create the singleton when they are first -# called. - -# Singleton object for use with module-level functions. The singleton is -# created as needed when one of the module-level functions is called. -_the_coverage = None - -def _singleton_method(name): - """Return a function to the `name` method on a singleton `Coverage` object. - - The singleton object is created the first time one of these functions is - called. - - """ - # Disable pylint message, because a bunch of variables look unused, but - # they're accessed via locals(). - # pylint: disable=unused-variable - - def wrapper(*args, **kwargs): - """Singleton wrapper around a coverage method.""" - global _the_coverage - if not _the_coverage: - _the_coverage = Coverage(auto_data=True) - return getattr(_the_coverage, name)(*args, **kwargs) - - import inspect - meth = getattr(Coverage, name) - args, varargs, kw, defaults = inspect.getargspec(meth) - argspec = inspect.formatargspec(args[1:], varargs, kw, defaults) - docstring = meth.__doc__ - wrapper.__doc__ = ("""\ - A first-use-singleton wrapper around Coverage.%(name)s. - - This wrapper is provided for backward compatibility with legacy code. - New code should use Coverage.%(name)s directly. - - %(name)s%(argspec)s: - - %(docstring)s - """ % locals() - ) - - return wrapper - - -# Define the module-level functions. -use_cache = _singleton_method('use_cache') -start = _singleton_method('start') -stop = _singleton_method('stop') -erase = _singleton_method('erase') -exclude = _singleton_method('exclude') -analysis = _singleton_method('analysis') -analysis2 = _singleton_method('analysis2') -report = _singleton_method('report') -annotate = _singleton_method('annotate') - - # On Windows, we encode and decode deep enough that something goes wrong and # the encodings.utf_8 module is loaded and then unloaded, I don't know why. # Adding a reference here prevents it from being unloaded. Yuk. @@ -91,34 +32,3 @@ try: del sys.modules['coverage.coverage'] except KeyError: pass - - -# COPYRIGHT AND LICENSE -# -# Copyright 2001 Gareth Rees. All rights reserved. -# Copyright 2004-2015 Ned Batchelder. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the -# distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -# DAMAGE. diff --git a/coverage/__main__.py b/coverage/__main__.py index 55e0d259..35ab87a5 100644 --- a/coverage/__main__.py +++ b/coverage/__main__.py @@ -1,4 +1,8 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Coverage.py's main entry point.""" + import sys from coverage.cmdline import main sys.exit(main()) diff --git a/coverage/annotate.py b/coverage/annotate.py index 6e68d4a3..4060450f 100644 --- a/coverage/annotate.py +++ b/coverage/annotate.py @@ -1,10 +1,19 @@ -"""Source file annotation for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt -import os, re +"""Source file annotation for coverage.py.""" -from coverage import env +import io +import os +import re + +from coverage.files import flat_rootname +from coverage.misc import isolate_module from coverage.report import Reporter +os = isolate_module(os) + + class AnnotateReporter(Reporter): """Generate annotated source files showing line coverage. @@ -53,14 +62,14 @@ class AnnotateReporter(Reporter): excluded = sorted(analysis.excluded) if self.directory: - dest_file = os.path.join(self.directory, fr.flat_rootname()) + dest_file = os.path.join(self.directory, flat_rootname(fr.relative_filename())) if dest_file.endswith("_py"): dest_file = dest_file[:-3] + ".py" dest_file += ",cover" else: dest_file = fr.filename + ",cover" - with open(dest_file, 'w') as dest: + with io.open(dest_file, 'w', encoding='utf8') as dest: i = 0 j = 0 covered = True @@ -73,25 +82,22 @@ class AnnotateReporter(Reporter): if i < len(statements) and statements[i] == lineno: covered = j >= len(missing) or missing[j] > lineno if self.blank_re.match(line): - dest.write(' ') + dest.write(u' ') elif self.else_re.match(line): # Special logic for lines containing only 'else:'. if i >= len(statements) and j >= len(missing): - dest.write('! ') + dest.write(u'! ') elif i >= len(statements) or j >= len(missing): - dest.write('> ') + dest.write(u'> ') elif statements[i] == missing[j]: - dest.write('! ') + dest.write(u'! ') else: - dest.write('> ') + dest.write(u'> ') elif lineno in excluded: - dest.write('- ') + dest.write(u'- ') elif covered: - dest.write('> ') + dest.write(u'> ') else: - dest.write('! ') - - if env.PY2: - line = line.encode('utf-8') + dest.write(u'! ') dest.write(line) diff --git a/coverage/backunittest.py b/coverage/backunittest.py index 95b6fcc6..09574ccb 100644 --- a/coverage/backunittest.py +++ b/coverage/backunittest.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Implementations of unittest features from the future.""" # Use unittest2 if it's available, otherwise unittest. This gives us @@ -22,10 +25,13 @@ class TestCase(unittest.TestCase): """ # pylint: disable=missing-docstring - if not unittest_has('assertCountEqual'): - def assertCountEqual(self, s1, s2): - """Assert these have the same elements, regardless of order.""" - self.assertEqual(set(s1), set(s2)) + # Many Pythons have this method defined. But PyPy3 has a bug with it + # somehow (https://bitbucket.org/pypy/pypy/issues/2092), so always use our + # own implementation that works everywhere, at least for the ways we're + # calling it. + def assertCountEqual(self, s1, s2): + """Assert these have the same elements, regardless of order.""" + self.assertEqual(sorted(s1), sorted(s2)) if not unittest_has('assertRaisesRegex'): def assertRaisesRegex(self, *args, **kwargs): diff --git a/coverage/backward.py b/coverage/backward.py index 58d9cfea..4fc72215 100644 --- a/coverage/backward.py +++ b/coverage/backward.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Add things to old Pythons so I can pretend they are newer.""" # This file does lots of tricky stuff, so disable a bunch of pylint warnings. @@ -16,6 +19,12 @@ try: except ImportError: from io import StringIO +# In py3, ConfigParser was renamed to the more-standard configparser +try: + import configparser +except ImportError: + import ConfigParser as configparser + # What's a string called? try: string_class = basestring @@ -40,6 +49,15 @@ try: except NameError: range = range +# shlex.quote is new, but there's an undocumented implementation in "pipes", +# who knew!? +try: + from shlex import quote as shlex_quote +except ImportError: + # Useful function, available under a different (undocumented) name + # in Python versions earlier than 3.3. + from pipes import quote as shlex_quote + # A function to iterate listlessly over a dict's items. try: {}.iteritems @@ -133,11 +151,12 @@ except AttributeError: PYC_MAGIC_NUMBER = imp.get_magic() -def import_local_file(modname): +def import_local_file(modname, modfile=None): """Import a local file as a module. Opens a file in the current directory named `modname`.py, imports it - as `modname`, and returns the module object. + as `modname`, and returns the module object. `modfile` is the file to + import if it isn't in the current directory. """ try: @@ -145,7 +164,8 @@ def import_local_file(modname): except ImportError: SourceFileLoader = None - modfile = modname + '.py' + if modfile is None: + modfile = modname + '.py' if SourceFileLoader: mod = SourceFileLoader(modname, modfile).load_module() else: @@ -155,7 +175,6 @@ def import_local_file(modname): with open(modfile, 'r') as f: # pylint: disable=undefined-loop-variable - # (Using possibly undefined loop variable 'suff') mod = imp.load_module(modname, f, modfile, suff) return mod diff --git a/coverage/bytecode.py b/coverage/bytecode.py index d7304936..82929cef 100644 --- a/coverage/bytecode.py +++ b/coverage/bytecode.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Bytecode manipulation for coverage.py""" import opcode diff --git a/coverage/cmdline.py b/coverage/cmdline.py index 66a76fa6..221c18d6 100644 --- a/coverage/cmdline.py +++ b/coverage/cmdline.py @@ -1,9 +1,13 @@ -"""Command-line support for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Command-line support for coverage.py.""" import glob import optparse -import os +import os.path import sys +import textwrap import traceback from coverage import env @@ -16,110 +20,122 @@ class Opts(object): """A namespace class for individual options we'll build parsers from.""" append = optparse.make_option( - '-a', '--append', action='store_false', dest="erase_first", - help="Append coverage data to .coverage, otherwise it is started " - "clean with each run." - ) + '-a', '--append', action='store_true', + help="Append coverage data to .coverage, otherwise it is started clean with each run.", + ) branch = optparse.make_option( '', '--branch', action='store_true', - help="Measure branch coverage in addition to statement coverage." - ) + help="Measure branch coverage in addition to statement coverage.", + ) CONCURRENCY_CHOICES = [ "thread", "gevent", "greenlet", "eventlet", "multiprocessing", ] concurrency = optparse.make_option( '', '--concurrency', action='store', metavar="LIB", choices=CONCURRENCY_CHOICES, - help="Properly measure code using a concurrency library. " - "Valid values are: %s." % ", ".join(CONCURRENCY_CHOICES) - ) + help=( + "Properly measure code using a concurrency library. " + "Valid values are: %s." + ) % ", ".join(CONCURRENCY_CHOICES), + ) debug = optparse.make_option( '', '--debug', action='store', metavar="OPTS", - help="Debug options, separated by commas" - ) + help="Debug options, separated by commas", + ) directory = optparse.make_option( '-d', '--directory', action='store', metavar="DIR", - help="Write the output files to DIR." - ) + help="Write the output files to DIR.", + ) fail_under = optparse.make_option( '', '--fail-under', action='store', metavar="MIN", type="int", - help="Exit with a status of 2 if the total coverage is less than MIN." - ) + help="Exit with a status of 2 if the total coverage is less than MIN.", + ) help = optparse.make_option( '-h', '--help', action='store_true', - help="Get help on this command." - ) + help="Get help on this command.", + ) ignore_errors = optparse.make_option( '-i', '--ignore-errors', action='store_true', - help="Ignore errors while reading source files." - ) + help="Ignore errors while reading source files.", + ) include = optparse.make_option( '', '--include', action='store', metavar="PAT1,PAT2,...", - help="Include only files whose paths match one of these patterns." - "Accepts shell-style wildcards, which must be quoted." - ) + help=( + "Include only files whose paths match one of these patterns. " + "Accepts shell-style wildcards, which must be quoted." + ), + ) pylib = optparse.make_option( '-L', '--pylib', action='store_true', - help="Measure coverage even inside the Python installed library, " - "which isn't done by default." - ) + help=( + "Measure coverage even inside the Python installed library, " + "which isn't done by default." + ), + ) show_missing = optparse.make_option( '-m', '--show-missing', action='store_true', - help="Show line numbers of statements in each module that weren't " - "executed." - ) + help="Show line numbers of statements in each module that weren't executed.", + ) skip_covered = optparse.make_option( '--skip-covered', action='store_true', - help="Skip files with 100% coverage." - ) + help="Skip files with 100% coverage.", + ) omit = optparse.make_option( '', '--omit', action='store', metavar="PAT1,PAT2,...", - help="Omit files whose paths match one of these patterns. " - "Accepts shell-style wildcards, which must be quoted." - ) + help=( + "Omit files whose paths match one of these patterns. " + "Accepts shell-style wildcards, which must be quoted." + ), + ) output_xml = optparse.make_option( '-o', '', action='store', dest="outfile", metavar="OUTFILE", - help="Write the XML report to this file. Defaults to 'coverage.xml'" - ) + help="Write the XML report to this file. Defaults to 'coverage.xml'", + ) parallel_mode = optparse.make_option( '-p', '--parallel-mode', action='store_true', - help="Append the machine name, process id and random number to the " - ".coverage data file name to simplify collecting data from " - "many processes." - ) + help=( + "Append the machine name, process id and random number to the " + ".coverage data file name to simplify collecting data from " + "many processes." + ), + ) module = optparse.make_option( '-m', '--module', action='store_true', - help="<pyfile> is an importable Python module, not a script path, " - "to be run as 'python -m' would run it." - ) + help=( + "<pyfile> is an importable Python module, not a script path, " + "to be run as 'python -m' would run it." + ), + ) rcfile = optparse.make_option( '', '--rcfile', action='store', - help="Specify configuration file. Defaults to '.coveragerc'" - ) + help="Specify configuration file. Defaults to '.coveragerc'", + ) source = optparse.make_option( '', '--source', action='store', metavar="SRC1,SRC2,...", - help="A list of packages or directories of code to be measured." - ) + help="A list of packages or directories of code to be measured.", + ) timid = optparse.make_option( '', '--timid', action='store_true', - help="Use a simpler but slower trace method. Try this if you get " - "seemingly impossible results!" - ) + help=( + "Use a simpler but slower trace method. Try this if you get " + "seemingly impossible results!" + ), + ) title = optparse.make_option( '', '--title', action='store', metavar="TITLE", - help="A text string to use as the title on the HTML." - ) + help="A text string to use as the title on the HTML.", + ) version = optparse.make_option( '', '--version', action='store_true', - help="Display version information and exit." - ) + help="Display version information and exit.", + ) class CoverageOptionParser(optparse.OptionParser, object): - """Base OptionParser for coverage. + """Base OptionParser for coverage.py. Problems don't exit the program. Defaults are initialized for all options. @@ -132,6 +148,7 @@ class CoverageOptionParser(optparse.OptionParser, object): ) self.set_defaults( action=None, + append=None, branch=None, concurrency=None, debug=None, @@ -140,9 +157,9 @@ class CoverageOptionParser(optparse.OptionParser, object): help=None, ignore_errors=None, include=None, + module=None, omit=None, parallel_mode=None, - module=None, pylib=None, rcfile=True, show_missing=None, @@ -150,7 +167,6 @@ class CoverageOptionParser(optparse.OptionParser, object): source=None, timid=None, title=None, - erase_first=None, version=None, ) @@ -199,10 +215,8 @@ class GlobalOptionParser(CoverageOptionParser): class CmdOptionParser(CoverageOptionParser): """Parse one of the new-style commands for coverage.py.""" - def __init__(self, action, options=None, defaults=None, usage=None, - description=None - ): - """Create an OptionParser for a coverage command. + def __init__(self, action, options=None, defaults=None, usage=None, description=None): + """Create an OptionParser for a coverage.py command. `action` is the slug to put into `options.action`. `options` is a list of Option's for the command. @@ -214,7 +228,6 @@ class CmdOptionParser(CoverageOptionParser): if usage: usage = "%prog " + usage super(CmdOptionParser, self).__init__( - prog="coverage %s" % action, usage=usage, description=description, ) @@ -228,6 +241,14 @@ class CmdOptionParser(CoverageOptionParser): # results, and they will compare equal to objects. return (other == "<CmdOptionParser:%s>" % self.cmd) + def get_prog_name(self): + """Override of an undocumented function in optparse.OptionParser.""" + program_name = super(CmdOptionParser, self).get_prog_name() + + # Include the sub-command for this parser as part of the command. + return "%(command)s %(subcommand)s" % {'command': program_name, 'subcommand': self.cmd} + + GLOBAL_ARGS = [ Opts.debug, Opts.help, @@ -235,115 +256,131 @@ GLOBAL_ARGS = [ ] CMDS = { - 'annotate': CmdOptionParser("annotate", + 'annotate': CmdOptionParser( + "annotate", [ Opts.directory, Opts.ignore_errors, - Opts.omit, Opts.include, + Opts.omit, ] + GLOBAL_ARGS, - usage = "[options] [modules]", - description = "Make annotated copies of the given files, marking " - "statements that are executed with > and statements that are " - "missed with !." + usage="[options] [modules]", + description=( + "Make annotated copies of the given files, marking statements that are executed " + "with > and statements that are missed with !." ), - - 'combine': CmdOptionParser("combine", GLOBAL_ARGS, - usage = "<dir1> <dir2> ... <dirN>", - description = "Combine data from multiple coverage files collected " + ), + + 'combine': CmdOptionParser( + "combine", + GLOBAL_ARGS, + usage="<path1> <path2> ... <pathN>", + description=( + "Combine data from multiple coverage files collected " "with 'run -p'. The combined results are written to a single " "file representing the union of the data. The positional " - "arguments are directories from which the data files should be " - "combined. By default, only data files in the current directory " - "are combined." + "arguments are data files or directories containing data files. " + "If no paths are provided, data files in the default data file's " + "directory are combined." ), + ), - 'debug': CmdOptionParser("debug", GLOBAL_ARGS, - usage = "<topic>", - description = "Display information on the internals of coverage.py, " + 'debug': CmdOptionParser( + "debug", GLOBAL_ARGS, + usage="<topic>", + description=( + "Display information on the internals of coverage.py, " "for diagnosing problems. " "Topics are 'data' to show a summary of the collected data, " "or 'sys' to show installation information." ), - - 'erase': CmdOptionParser("erase", GLOBAL_ARGS, - usage = " ", - description = "Erase previously collected coverage data." - ), - - 'help': CmdOptionParser("help", GLOBAL_ARGS, - usage = "[command]", - description = "Describe how to use coverage.py" - ), - - 'html': CmdOptionParser("html", + ), + + 'erase': CmdOptionParser( + "erase", GLOBAL_ARGS, + usage=" ", + description="Erase previously collected coverage data.", + ), + + 'help': CmdOptionParser( + "help", GLOBAL_ARGS, + usage="[command]", + description="Describe how to use coverage.py", + ), + + 'html': CmdOptionParser( + "html", [ Opts.directory, Opts.fail_under, Opts.ignore_errors, - Opts.omit, Opts.include, + Opts.omit, Opts.title, ] + GLOBAL_ARGS, - usage = "[options] [modules]", - description = "Create an HTML report of the coverage of the files. " + usage="[options] [modules]", + description=( + "Create an HTML report of the coverage of the files. " "Each file gets its own page, with the source decorated to show " "executed, excluded, and missed lines." ), + ), - 'report': CmdOptionParser("report", + 'report': CmdOptionParser( + "report", [ Opts.fail_under, Opts.ignore_errors, - Opts.omit, Opts.include, + Opts.omit, Opts.show_missing, - Opts.skip_covered + Opts.skip_covered, ] + GLOBAL_ARGS, - usage = "[options] [modules]", - description = "Report coverage statistics on modules." - ), + usage="[options] [modules]", + description="Report coverage statistics on modules." + ), - 'run': CmdOptionParser("run", + 'run': CmdOptionParser( + "run", [ Opts.append, Opts.branch, Opts.concurrency, + Opts.include, + Opts.module, + Opts.omit, Opts.pylib, Opts.parallel_mode, - Opts.module, - Opts.timid, Opts.source, - Opts.omit, - Opts.include, + Opts.timid, ] + GLOBAL_ARGS, - defaults = {'erase_first': True}, - usage = "[options] <pyfile> [program options]", - description = "Run a Python program, measuring code execution." - ), + usage="[options] <pyfile> [program options]", + description="Run a Python program, measuring code execution." + ), - 'xml': CmdOptionParser("xml", + 'xml': CmdOptionParser( + "xml", [ Opts.fail_under, Opts.ignore_errors, - Opts.omit, Opts.include, + Opts.omit, Opts.output_xml, ] + GLOBAL_ARGS, - usage = "[options] [modules]", - description = "Generate an XML report of coverage results." - ), - } + usage="[options] [modules]", + description="Generate an XML report of coverage results." + ), +} OK, ERR, FAIL_UNDER = 0, 1, 2 class CoverageScript(object): - """The command-line interface to Coverage.""" + """The command-line interface to coverage.py.""" def __init__(self, _covpkg=None, _run_python_file=None, - _run_python_module=None, _help_fn=None): + _run_python_module=None, _help_fn=None, _path_exists=None): # _covpkg is for dependency injection, so we can test this code. if _covpkg: self.covpkg = _covpkg @@ -355,12 +392,24 @@ class CoverageScript(object): self.run_python_file = _run_python_file or run_python_file self.run_python_module = _run_python_module or run_python_module self.help_fn = _help_fn or self.help + self.path_exists = _path_exists or os.path.exists self.global_option = False self.coverage = None + self.program_name = os.path.basename(sys.argv[0]) + if env.WINDOWS: + # entry_points={'console_scripts':...} on Windows makes files + # called coverage.exe, coverage3.exe, and coverage-3.5.exe. These + # invoke coverage-script.py, coverage3-script.py, and + # coverage-3.5-script.py. argv[0] is the .py file, but we want to + # get back to the original form. + auto_suffix = "-script.py" + if self.program_name.endswith(auto_suffix): + self.program_name = self.program_name[:-len(auto_suffix)] + def command_line(self, argv): - """The bulk of the command line interface to Coverage. + """The bulk of the command line interface to coverage.py. `argv` is the argument list to process. @@ -409,55 +458,58 @@ class CoverageScript(object): # Do something. self.coverage = self.covpkg.coverage( - data_suffix = options.parallel_mode, - cover_pylib = options.pylib, - timid = options.timid, - branch = options.branch, - config_file = options.rcfile, - source = source, - omit = omit, - include = include, - debug = debug, - concurrency = options.concurrency, + data_suffix=options.parallel_mode, + cover_pylib=options.pylib, + timid=options.timid, + branch=options.branch, + config_file=options.rcfile, + source=source, + omit=omit, + include=include, + debug=debug, + concurrency=options.concurrency, ) if options.action == "debug": return self.do_debug(args) - if options.action == "erase" or options.erase_first: + elif options.action == "erase": self.coverage.erase() - else: - self.coverage.load() + return OK - if options.action == "run": - self.do_run(options, args) + elif options.action == "run": + return self.do_run(options, args) - if options.action == "combine": - data_dirs = argv if argv else None + elif options.action == "combine": + self.coverage.load() + data_dirs = args or None self.coverage.combine(data_dirs) self.coverage.save() + return OK # Remaining actions are reporting, with some common options. report_args = dict( - morfs = unglob_args(args), - ignore_errors = options.ignore_errors, - omit = omit, - include = include, + morfs=unglob_args(args), + ignore_errors=options.ignore_errors, + omit=omit, + include=include, ) + self.coverage.load() + total = None if options.action == "report": total = self.coverage.report( show_missing=options.show_missing, skip_covered=options.skip_covered, **report_args) - if options.action == "annotate": + elif options.action == "annotate": self.coverage.annotate( directory=options.directory, **report_args) - if options.action == "html": + elif options.action == "html": total = self.coverage.html_report( directory=options.directory, title=options.title, **report_args) - if options.action == "xml": + elif options.action == "xml": outfile = options.outfile total = self.coverage.xml_report(outfile=outfile, **report_args) @@ -465,9 +517,9 @@ class CoverageScript(object): # Apply the command line fail-under options, and then use the config # value, so we can get fail_under from the config file. if options.fail_under is not None: - self.coverage.config["report:fail_under"] = options.fail_under + self.coverage.set_option("report:fail_under", options.fail_under) - if self.coverage.config["report:fail_under"]: + if self.coverage.get_option("report:fail_under"): # Total needs to be rounded, but be careful of 0 and 100. if 0 < total < 1: @@ -477,7 +529,7 @@ class CoverageScript(object): else: total = round(total) - if total >= self.coverage.config["report:fail_under"]: + if total >= self.coverage.get_option("report:fail_under"): return OK else: return FAIL_UNDER @@ -489,13 +541,15 @@ class CoverageScript(object): assert error or topic or parser if error: print(error) - print("Use 'coverage help' for help.") + print("Use '%s help' for help." % (self.program_name,)) elif parser: print(parser.format_help().strip()) else: - help_msg = HELP_TOPICS.get(topic, '').strip() + help_params = dict(self.covpkg.__dict__) + help_params['program_name'] = self.program_name + help_msg = textwrap.dedent(HELP_TOPICS.get(topic, '')).strip() if help_msg: - print(help_msg % self.covpkg.__dict__) + print(help_msg % help_params) else: print("Don't know topic %r" % topic) @@ -547,19 +601,22 @@ class CoverageScript(object): def do_run(self, options, args): """Implementation of 'coverage run'.""" - # Set the first path element properly. - old_path0 = sys.path[0] + if options.append and self.coverage.get_option("run:parallel"): + self.help_fn("Can't append to data files in parallel mode.") + return ERR + + if not self.coverage.get_option("run:parallel"): + if not options.append: + self.coverage.erase() # Run the script. self.coverage.start() code_ran = True try: if options.module: - sys.path[0] = '' self.run_python_module(args[0], args) else: filename = args[0] - sys.path[0] = os.path.abspath(os.path.dirname(filename)) self.run_python_file(filename, args) except NoSource: code_ran = False @@ -567,10 +624,13 @@ class CoverageScript(object): finally: self.coverage.stop() if code_ran: + if options.append: + data_file = self.coverage.get_option("run:data_file") + if self.path_exists(data_file): + self.coverage.combine(data_paths=[data_file]) self.coverage.save() - # Restore the old path - sys.path[0] = old_path0 + return OK def do_debug(self, args): """Implementation of 'coverage debug'.""" @@ -578,6 +638,7 @@ class CoverageScript(object): if not args: self.help_fn("What information would you like: data, sys?") return ERR + for info in args: if info == 'sys': sys_info = self.coverage.sys_info() @@ -586,17 +647,17 @@ class CoverageScript(object): print(" %s" % line) elif info == 'data': self.coverage.load() + data = self.coverage.data print(info_header("data")) - print("path: %s" % self.coverage.data.filename) - print("has_arcs: %r" % self.coverage.data.has_arcs()) - summary = self.coverage.data.summary(fullpath=True) - if summary: - plugins = self.coverage.data.plugin_data() + print("path: %s" % self.coverage.data_files.filename) + if data: + print("has_arcs: %r" % data.has_arcs()) + summary = data.line_counts(fullpath=True) filenames = sorted(summary.keys()) print("\n%d files:" % len(filenames)) for f in filenames: line = "%s: %d lines" % (f, summary[f]) - plugin = plugins.get(f) + plugin = data.file_tracer(f) if plugin: line += " [%s]" % plugin print(line) @@ -605,6 +666,7 @@ class CoverageScript(object): else: self.help_fn("Don't know what you mean by %r" % info) return ERR + return OK @@ -613,9 +675,9 @@ def unshell_list(s): if not s: return None if env.WINDOWS: - # When running coverage as coverage.exe, some of the behavior + # When running coverage.py as coverage.exe, some of the behavior # of the shell is emulated: wildcards are expanded into a list of - # filenames. So you have to single-quote patterns on the command + # file names. So you have to single-quote patterns on the command # line, but (not) helpfully, the single quotes are included in the # argument, so we have to strip them off here. s = s.strip("'") @@ -636,40 +698,39 @@ def unglob_args(args): HELP_TOPICS = { -# ------------------------- -'help': """\ -Coverage.py, version %(__version__)s -Measure, collect, and report on code coverage in Python programs. - -usage: coverage <command> [options] [args] - -Commands: - annotate Annotate source files with execution information. - combine Combine a number of data files. - erase Erase previously collected coverage data. - help Get help on using coverage.py. - html Create an HTML report. - report Report coverage stats on modules. - run Run a Python program and measure code execution. - xml Create an XML report of coverage results. - -Use "coverage help <command>" for detailed help on any command. -For full documentation, see %(__url__)s -""", -# ------------------------- -'minimum_help': """\ -Code coverage for Python. Use 'coverage help' for help. -""", -# ------------------------- -'version': """\ -Coverage.py, version %(__version__)s. -Documentation at %(__url__)s -""", + 'help': """\ + Coverage.py, version %(__version__)s + Measure, collect, and report on code coverage in Python programs. + + usage: %(program_name)s <command> [options] [args] + + Commands: + annotate Annotate source files with execution information. + combine Combine a number of data files. + erase Erase previously collected coverage data. + help Get help on using coverage.py. + html Create an HTML report. + report Report coverage stats on modules. + run Run a Python program and measure code execution. + xml Create an XML report of coverage results. + + Use "%(program_name)s help <command>" for detailed help on any command. + For full documentation, see %(__url__)s + """, + + 'minimum_help': """\ + Code coverage for Python. Use '%(program_name)s help' for help. + """, + + 'version': """\ + Coverage.py, version %(__version__)s. + Documentation at %(__url__)s + """, } def main(argv=None): - """The main entry point to Coverage. + """The main entry point to coverage.py. This is installed as the script entry point. diff --git a/coverage/collector.py b/coverage/collector.py index 948cbbb0..0a43d87c 100644 --- a/coverage/collector.py +++ b/coverage/collector.py @@ -1,14 +1,23 @@ -"""Raw data collector for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt -import os, sys +"""Raw data collector for coverage.py.""" + +import os +import sys from coverage import env -from coverage.misc import CoverageException +from coverage.backward import iitems +from coverage.files import abs_file +from coverage.misc import CoverageException, isolate_module from coverage.pytracer import PyTracer +os = isolate_module(os) + + try: # Use the C extension code when we can, for speed. - from coverage.tracer import CTracer # pylint: disable=no-name-in-module + from coverage.tracer import CTracer, CFileDisposition # pylint: disable=no-name-in-module except ImportError: # Couldn't import the C extension, maybe it isn't built. if os.getenv('COVERAGE_TEST_TRACER') == 'c': @@ -18,13 +27,16 @@ except ImportError: # it, then exit quickly and clearly instead of dribbling confusing # errors. I'm using sys.exit here instead of an exception because an # exception here causes all sorts of other noise in unittest. - sys.stderr.write( - "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n" - ) + sys.stderr.write("*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n") sys.exit(1) CTracer = None +class FileDisposition(object): + """A simple value type for recording what to do with a file.""" + pass + + class Collector(object): """Collects trace data. @@ -46,16 +58,14 @@ class Collector(object): # the top, and resumed when they become the top again. _collectors = [] - def __init__(self, - should_trace, check_include, timid, branch, warn, concurrency, - ): + def __init__(self, should_trace, check_include, timid, branch, warn, concurrency): """Create a collector. - `should_trace` is a function, taking a filename, and returning a - canonicalized filename, or None depending on whether the file should - be traced or not. + `should_trace` is a function, taking a file name, and returning a + `coverage.FileDisposition object`. - TODO: `check_include` + `check_include` is a function taking a file name and a frame. It returns + a boolean: True if the file should be traced, False if not. If `timid` is true, then a slower simpler trace function will be used. This is important for some environments where manipulation of @@ -85,13 +95,13 @@ class Collector(object): try: if concurrency == "greenlet": - import greenlet # pylint: disable=import-error + import greenlet self.concur_id_func = greenlet.getcurrent elif concurrency == "eventlet": - import eventlet.greenthread # pylint: disable=import-error + import eventlet.greenthread # pylint: disable=import-error,useless-suppression self.concur_id_func = eventlet.greenthread.getcurrent elif concurrency == "gevent": - import gevent # pylint: disable=import-error + import gevent # pylint: disable=import-error,useless-suppression self.concur_id_func = gevent.getcurrent elif concurrency == "thread" or not concurrency: # It's important to import threading only if we need it. If @@ -100,13 +110,10 @@ class Collector(object): import threading self.threading = threading else: - raise CoverageException( - "Don't understand concurrency=%s" % concurrency - ) + raise CoverageException("Don't understand concurrency=%s" % concurrency) except ImportError: raise CoverageException( - "Couldn't trace with concurrency=%s, " - "the module isn't installed." % concurrency + "Couldn't trace with concurrency=%s, the module isn't installed." % concurrency ) self.reset() @@ -119,10 +126,15 @@ class Collector(object): # trace function. self._trace_class = CTracer or PyTracer - self.supports_plugins = self._trace_class is CTracer + if self._trace_class is CTracer: + self.file_disposition_class = CFileDisposition + self.supports_plugins = True + else: + self.file_disposition_class = FileDisposition + self.supports_plugins = False def __repr__(self): - return "<Collector at 0x%x>" % id(self) + return "<Collector at 0x%x: %s>" % (id(self), self.tracer_name()) def tracer_name(self): """Return the class name of the tracer we're using.""" @@ -130,15 +142,23 @@ class Collector(object): def reset(self): """Clear collected data, and prepare to collect more.""" - # A dictionary mapping filenames to dicts with line number keys, - # or mapping filenames to dicts with line number pairs as keys. + # A dictionary mapping file names to dicts with line number keys (if not + # branch coverage), or mapping file names to dicts with line number + # pairs as keys (if branch coverage). self.data = {} - self.plugin_data = {} - - # A cache of the results from should_trace, the decision about whether - # to trace execution in a file. A dict of filename to (filename or - # None). + # A dictionary mapping file names to file tracer plugin names that will + # handle them. + self.file_tracers = {} + + # The .should_trace_cache attribute is a cache from file names to + # coverage.FileDisposition objects, or None. When a file is first + # considered for tracing, a FileDisposition is obtained from + # Coverage.should_trace. Its .trace attribute indicates whether the + # file should be traced or not. If it should be, a plugin with dynamic + # file names can decide not to trace it based on the dynamic file name + # being excluded by the inclusion rules, in which case the + # FileDisposition will be replaced by None in the cache. if env.PYPY: import __pypy__ # pylint: disable=import-error # Alex Gaynor said: @@ -166,7 +186,7 @@ class Collector(object): """Start a new Tracer object, and store it in self.tracers.""" tracer = self._trace_class() tracer.data = self.data - tracer.arcs = self.branch + tracer.trace_arcs = self.branch tracer.should_trace = self.should_trace tracer.should_trace_cache = self.should_trace_cache tracer.warn = self.warn @@ -175,14 +195,13 @@ class Collector(object): tracer.concur_id_func = self.concur_id_func elif self.concur_id_func: raise CoverageException( - "Can't support concurrency=%s with %s, " - "only threads are supported" % ( + "Can't support concurrency=%s with %s, only threads are supported" % ( self.concurrency, self.tracer_name(), ) ) - if hasattr(tracer, 'plugin_data'): - tracer.plugin_data = self.plugin_data + if hasattr(tracer, 'file_tracers'): + tracer.file_tracers = self.file_tracers if hasattr(tracer, 'threading'): tracer.threading = self.threading if hasattr(tracer, 'check_include'): @@ -199,16 +218,16 @@ class Collector(object): # install this as a trace function, and the first time it's called, it does # the real trace installation. - def _installation_trace(self, frame_unused, event_unused, arg_unused): + def _installation_trace(self, frame, event, arg): """Called on new threads, installs the real tracer.""" - # Remove ourselves as the trace function + # Remove ourselves as the trace function. sys.settrace(None) # Install the real tracer. fn = self._start_tracer() # Invoke the real trace function with the current event, to be sure # not to lose an event. if fn: - fn = fn(frame_unused, event_unused, arg_unused) + fn = fn(frame, event, arg) # Return the new trace function to continue tracing in this scope. return fn @@ -216,9 +235,9 @@ class Collector(object): """Start collecting trace information.""" if self._collectors: self._collectors[-1].pause() - self._collectors.append(self) - # Check to see whether we had a fullcoverage tracer installed. + # Check to see whether we had a fullcoverage tracer installed. If so, + # get the stack frames it stashed away for us. traces0 = [] fn0 = sys.gettrace() if fn0: @@ -226,8 +245,17 @@ class Collector(object): if tracer0: traces0 = getattr(tracer0, 'traces', []) - # Install the tracer on this thread. - fn = self._start_tracer() + try: + # Install the tracer on this thread. + fn = self._start_tracer() + except: + if self._collectors: + self._collectors[-1].resume() + raise + + # If _start_tracer succeeded, then we add ourselves to the global + # stack of collectors. + self._collectors.append(self) # Replay all the events from fullcoverage into the new trace function. for args in traces0: @@ -235,9 +263,7 @@ class Collector(object): try: fn(frame, event, arg, lineno=lineno) except TypeError: - raise Exception( - "fullcoverage must be run with the C trace function." - ) + raise Exception("fullcoverage must be run with the C trace function.") # Install our installation tracer in threading, to jump start other # threads. @@ -248,9 +274,7 @@ class Collector(object): """Stop collecting trace information.""" assert self._collectors assert self._collectors[-1] is self, ( - "Expected current collector to be %r, but it's %r" % ( - self, self._collectors[-1], - ) + "Expected current collector to be %r, but it's %r" % (self, self._collectors[-1]) ) self.pause() @@ -283,39 +307,20 @@ class Collector(object): else: self._start_tracer() - def get_line_data(self): - """Return the line data collected. + def save_data(self, covdata): + """Save the collected data to a `CoverageData`. - Data is { filename: { lineno: None, ...}, ...} + Also resets the collector. """ - if self.branch: - # If we were measuring branches, then we have to re-build the dict - # to show line data. We'll use the first lines of all the arcs, - # if they are actual lines. We don't need the second lines, because - # the second lines will also be first lines, sometimes to exits. - line_data = {} - for f, arcs in self.data.items(): - line_data[f] = dict( - (l1, None) for l1, _ in arcs.keys() if l1 > 0 - ) - return line_data - else: - return self.data + def abs_file_dict(d): + """Return a dict like d, but with keys modified by `abs_file`.""" + return dict((abs_file(k), v) for k, v in iitems(d)) - def get_arc_data(self): - """Return the arc data collected. - - Data is { filename: { (l1, l2): None, ...}, ...} - - Note that no data is collected or returned if the Collector wasn't - created with `branch` true. - - """ if self.branch: - return self.data + covdata.add_arcs(abs_file_dict(self.data)) else: - return {} + covdata.add_lines(abs_file_dict(self.data)) + covdata.add_file_tracers(abs_file_dict(self.file_tracers)) - def get_plugin_data(self): - return self.plugin_data + self.reset() diff --git a/coverage/config.py b/coverage/config.py index 7b142671..cd66697d 100644 --- a/coverage/config.py +++ b/coverage/config.py @@ -1,27 +1,28 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Config file for coverage.py""" -import os, re, sys -from coverage.backward import string_class, iitems -from coverage.misc import CoverageException +import collections +import os +import re +import sys +from coverage.backward import configparser, iitems, string_class +from coverage.misc import CoverageException, isolate_module -# In py3, ConfigParser was renamed to the more-standard configparser -try: - import configparser -except ImportError: - import ConfigParser as configparser +os = isolate_module(os) class HandyConfigParser(configparser.RawConfigParser): """Our specialization of ConfigParser.""" def __init__(self, section_prefix): - # pylint: disable=super-init-not-called configparser.RawConfigParser.__init__(self) self.section_prefix = section_prefix def read(self, filename): - """Read a filename as UTF-8 configuration data.""" + """Read a file name as UTF-8 configuration data.""" kwargs = {} if sys.version_info >= (3, 2): kwargs['encoding'] = "utf-8" @@ -61,7 +62,7 @@ class HandyConfigParser(configparser.RawConfigParser): def dollar_replace(m): """Called for each $replacement.""" # Only one of the groups will have matched, just get its text. - word = next(w for w in m.groups() if w is not None) + word = next(w for w in m.groups() if w is not None) # pragma: part covered if word == "$": return "$" else: @@ -112,10 +113,8 @@ class HandyConfigParser(configparser.RawConfigParser): re.compile(value) except re.error as e: raise CoverageException( - "Invalid [%s].%s value %r: %s" % ( - section, option, value, e - ) - ) + "Invalid [%s].%s value %r: %s" % (section, option, value, e) + ) if value: value_list.append(value) return value_list @@ -124,12 +123,12 @@ class HandyConfigParser(configparser.RawConfigParser): # The default line exclusion regexes. DEFAULT_EXCLUDE = [ r'(?i)#\s*pragma[:\s]?\s*no\s*cover', - ] +] # The default partial branch regexes, to be modified by the user. DEFAULT_PARTIAL = [ r'(?i)#\s*pragma[:\s]?\s*no\s*branch', - ] +] # The default partial branch regexes, based on Python semantics. # These are any Python branching constructs that can't actually execute all @@ -137,7 +136,7 @@ DEFAULT_PARTIAL = [ DEFAULT_PARTIAL_ALWAYS = [ 'while (True|1|False|0):', 'if (True|1|False|0):', - ] +] class CoverageConfig(object): @@ -158,11 +157,12 @@ class CoverageConfig(object): self.concurrency = None self.cover_pylib = False self.data_file = ".coverage" - self.parallel = False - self.timid = False - self.source = None self.debug = [] + self.note = None + self.parallel = False self.plugins = [] + self.source = None + self.timid = False # Defaults for [report] self.exclude_list = DEFAULT_EXCLUDE[:] @@ -170,15 +170,15 @@ class CoverageConfig(object): self.ignore_errors = False self.include = None self.omit = None - self.partial_list = DEFAULT_PARTIAL[:] self.partial_always_list = DEFAULT_PARTIAL_ALWAYS[:] + self.partial_list = DEFAULT_PARTIAL[:] self.precision = 0 self.show_missing = False self.skip_covered = False # Defaults for [html] - self.html_dir = "htmlcov" self.extra_css = None + self.html_dir = "htmlcov" self.html_title = "Coverage report" # Defaults for [xml] @@ -215,9 +215,7 @@ class CoverageConfig(object): try: files_read = cp.read(filename) except configparser.Error as err: - raise CoverageException( - "Couldn't read config file %s: %s" % (filename, err) - ) + raise CoverageException("Couldn't read config file %s: %s" % (filename, err)) if not files_read: return False @@ -227,9 +225,24 @@ class CoverageConfig(object): for option_spec in self.CONFIG_FILE_OPTIONS: self._set_attr_from_config_option(cp, *option_spec) except ValueError as err: - raise CoverageException( - "Couldn't read config file %s: %s" % (filename, err) - ) + raise CoverageException("Couldn't read config file %s: %s" % (filename, err)) + + # Check that there are no unrecognized options. + all_options = collections.defaultdict(set) + for option_spec in self.CONFIG_FILE_OPTIONS: + section, option = option_spec[1].split(":") + all_options[section].add(option) + + for section, options in iitems(all_options): + if cp.has_section(section): + for unknown in set(cp.options(section)) - options: + if section_prefix: + section = section_prefix + section + raise CoverageException( + "Unrecognized option '[%s] %s=' in config file %s" % ( + section, unknown, filename + ) + ) # [paths] is special if cp.has_section('paths'): @@ -258,10 +271,11 @@ class CoverageConfig(object): ('cover_pylib', 'run:cover_pylib', 'boolean'), ('data_file', 'run:data_file'), ('debug', 'run:debug', 'list'), - ('plugins', 'run:plugins', 'list'), ('include', 'run:include', 'list'), + ('note', 'run:note'), ('omit', 'run:omit', 'list'), ('parallel', 'run:parallel', 'boolean'), + ('plugins', 'run:plugins', 'list'), ('source', 'run:source', 'list'), ('timid', 'run:timid', 'boolean'), @@ -271,35 +285,44 @@ class CoverageConfig(object): ('ignore_errors', 'report:ignore_errors', 'boolean'), ('include', 'report:include', 'list'), ('omit', 'report:omit', 'list'), - ('partial_list', 'report:partial_branches', 'regexlist'), ('partial_always_list', 'report:partial_branches_always', 'regexlist'), + ('partial_list', 'report:partial_branches', 'regexlist'), ('precision', 'report:precision', 'int'), ('show_missing', 'report:show_missing', 'boolean'), ('skip_covered', 'report:skip_covered', 'boolean'), # [html] - ('html_dir', 'html:directory'), ('extra_css', 'html:extra_css'), + ('html_dir', 'html:directory'), ('html_title', 'html:title'), # [xml] ('xml_output', 'xml:output'), ('xml_package_depth', 'xml:package_depth', 'int'), - ] + ] def _set_attr_from_config_option(self, cp, attr, where, type_=''): """Set an attribute on self if it exists in the ConfigParser.""" section, option = where.split(":") if cp.has_option(section, option): - method = getattr(cp, 'get'+type_) + method = getattr(cp, 'get' + type_) setattr(self, attr, method(section, option)) def get_plugin_options(self, plugin): """Get a dictionary of options for the plugin named `plugin`.""" return self.plugin_options.get(plugin, {}) - # TODO: docs for this. - def __setitem__(self, option_name, value): + def set_option(self, option_name, value): + """Set an option in the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with `"run:branch"`. + + `value` is the new value for the option. + + """ + # Check all the hard-coded options. for option_spec in self.CONFIG_FILE_OPTIONS: attr, where = option_spec[:2] @@ -316,8 +339,17 @@ class CoverageConfig(object): # If we get here, we didn't find the option. raise CoverageException("No such option: %r" % option_name) - # TODO: docs for this. - def __getitem__(self, option_name): + def get_option(self, option_name): + """Get an option from the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with `"run:branch"`. + + Returns the value of the option. + + """ + # Check all the hard-coded options. for option_spec in self.CONFIG_FILE_OPTIONS: attr, where = option_spec[:2] diff --git a/coverage/control.py b/coverage/control.py index 2c8d384e..0a5ccae6 100644 --- a/coverage/control.py +++ b/coverage/control.py @@ -1,34 +1,38 @@ -"""Core control stuff for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Core control stuff for coverage.py.""" import atexit import inspect import os import platform -import random -import socket +import re import sys import traceback -from coverage import env +from coverage import env, files from coverage.annotate import AnnotateReporter from coverage.backward import string_class, iitems from coverage.collector import Collector from coverage.config import CoverageConfig -from coverage.data import CoverageData +from coverage.data import CoverageData, CoverageDataFiles from coverage.debug import DebugControl -from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher +from coverage.files import TreeMatcher, FnmatchMatcher from coverage.files import PathAliases, find_python_files, prep_patterns from coverage.files import ModuleMatcher, abs_file from coverage.html import HtmlReporter from coverage.misc import CoverageException, bool_or_none, join_regex -from coverage.misc import file_be_gone, overrides +from coverage.misc import file_be_gone, isolate_module from coverage.monkey import patch_multiprocessing -from coverage.plugin import CoveragePlugin, FileReporter +from coverage.plugin import FileReporter +from coverage.plugin_support import Plugins from coverage.python import PythonFileReporter from coverage.results import Analysis, Numbers from coverage.summary import SummaryReporter from coverage.xmlreport import XmlReporter +os = isolate_module(os) # Pypy has some unusual stuff in the "stdlib". Consider those locations # when deciding where the stdlib is. @@ -97,7 +101,7 @@ class Coverage(object): in the trees indicated by the file paths or package names will be measured. - `include` and `omit` are lists of filename patterns. Files that match + `include` and `omit` are lists of file name patterns. Files that match `include` will be measured, files that match `omit` will not. Each will also accept a single string argument. @@ -109,6 +113,9 @@ class Coverage(object): results. Valid strings are "greenlet", "eventlet", "gevent", or "thread" (the default). + .. versionadded:: 4.0 + The `concurrency` parameter. + """ # Build our configuration from a number of sources: # 1: defaults: @@ -138,6 +145,9 @@ class Coverage(object): env_data_file = os.environ.get('COVERAGE_FILE') if env_data_file: self.config.data_file = env_data_file + debugs = os.environ.get('COVERAGE_DEBUG') + if debugs: + self.config.debug.extend(debugs.split(",")) # 4: from constructor arguments: self.config.from_args( @@ -166,10 +176,10 @@ class Coverage(object): # Other instance attributes, set later. self.omit = self.include = self.source = None - self.source_pkgs = self.file_locator = None - self.data = self.collector = None - self.plugins = self.file_tracing_plugins = None - self.pylib_dirs = self.cover_dir = None + self.source_pkgs = None + self.data = self.data_files = self.collector = None + self.plugins = None + self.pylib_dirs = self.cover_dirs = None self.data_suffix = self.run_suffix = None self._exclude_re = None self.debug = None @@ -186,41 +196,40 @@ class Coverage(object): """Set all the initial state. This is called by the public methods to initialize state. This lets us - construct a Coverage object, then tweak its state before this function - is called. + construct a :class:`Coverage` object, then tweak its state before this + function is called. """ - from coverage import __version__ - if self._inited: return - # Create and configure the debugging controller. + # Create and configure the debugging controller. COVERAGE_DEBUG_FILE + # is an environment variable, the name of a file to append debug logs + # to. if self._debug_file is None: - self._debug_file = sys.stderr + debug_file_name = os.environ.get("COVERAGE_DEBUG_FILE") + if debug_file_name: + self._debug_file = open(debug_file_name, "a") + else: + self._debug_file = sys.stderr self.debug = DebugControl(self.config.debug, self._debug_file) # Load plugins - self.plugins = Plugins.load_plugins(self.config.plugins, self.config) - - self.file_tracing_plugins = [] - for plugin in self.plugins: - if overrides(plugin, "file_tracer", CoveragePlugin): - self.file_tracing_plugins.append(plugin) + self.plugins = Plugins.load_plugins(self.config.plugins, self.config, self.debug) # _exclude_re is a dict that maps exclusion list names to compiled # regexes. self._exclude_re = {} self._exclude_regex_stale() - self.file_locator = FileLocator() + files.set_relative_directory() # The source argument can be directories or package names. self.source = [] self.source_pkgs = [] for src in self.config.source or []: if os.path.exists(src): - self.source.append(self.file_locator.canonical_filename(src)) + self.source.append(files.canonical_filename(src)) else: self.source_pkgs.append(src) @@ -242,17 +251,17 @@ class Coverage(object): ) # Early warning if we aren't going to be able to support plugins. - if self.file_tracing_plugins and not self.collector.supports_plugins: + if self.plugins.file_tracers and not self.collector.supports_plugins: self._warn( "Plugin file tracers (%s) aren't supported with %s" % ( ", ".join( plugin._coverage_plugin_name - for plugin in self.file_tracing_plugins + for plugin in self.plugins.file_tracers ), self.collector.tracer_name(), ) ) - for plugin in self.file_tracing_plugins: + for plugin in self.plugins.file_tracers: plugin._coverage_enabled = False # Suffixes are a bit tricky. We want to use the data suffix only when @@ -271,13 +280,10 @@ class Coverage(object): # Create the data file. We do this at construction time so that the # data file will be written into the directory where the process # started rather than wherever the process eventually chdir'd to. - self.data = CoverageData( - basename=self.config.data_file, - collector="coverage v%s" % __version__, - debug=self.debug, - ) + self.data = CoverageData(debug=self.debug) + self.data_files = CoverageDataFiles(basename=self.config.data_file, warn=self._warn) - # The dirs for files considered "installed with the interpreter". + # The directories for files considered "installed with the interpreter". self.pylib_dirs = set() if not self.config.cover_pylib: # Look at where some standard modules are located. That's the @@ -285,12 +291,12 @@ class Coverage(object): # environments (virtualenv, for example), these modules may be # spread across a few locations. Look at all the candidate modules # we've imported, and take all the different ones. - for m in (atexit, os, platform, random, socket, _structseq): + for m in (atexit, inspect, os, platform, re, _structseq, traceback): if m is not None and hasattr(m, "__file__"): self.pylib_dirs.add(self._canonical_dir(m)) if _structseq and not hasattr(_structseq, '__file__'): # PyPy 2.4 has no __file__ in the builtin modules, but the code - # objects still have the filenames. So dig into one to find + # objects still have the file names. So dig into one to find # the path to exclude. structseq_new = _structseq.structseq_new try: @@ -299,9 +305,16 @@ class Coverage(object): structseq_file = structseq_new.__code__.co_filename self.pylib_dirs.add(self._canonical_dir(structseq_file)) - # To avoid tracing the coverage code itself, we skip anything located - # where we are. - self.cover_dir = self._canonical_dir(__file__) + # To avoid tracing the coverage.py code itself, we skip anything + # located where we are. + self.cover_dirs = [self._canonical_dir(__file__)] + if env.TESTING: + # When testing, we use PyContracts, which should be considered + # part of coverage.py, and it uses six. Exclude those directories + # just as we exclude ourselves. + import contracts, six + for mod in [contracts, six]: + self.cover_dirs.append(self._canonical_dir(mod)) # Set the reporting precision. Numbers.set_precision(self.config.precision) @@ -315,8 +328,8 @@ class Coverage(object): self.source_match = TreeMatcher(self.source) self.source_pkgs_match = ModuleMatcher(self.source_pkgs) else: - if self.cover_dir: - self.cover_match = TreeMatcher([self.cover_dir]) + if self.cover_dirs: + self.cover_match = TreeMatcher(self.cover_dirs) if self.pylib_dirs: self.pylib_match = TreeMatcher(self.pylib_dirs) if self.include: @@ -350,7 +363,7 @@ class Coverage(object): def _source_for_file(self, filename): """Return the source file for `filename`. - Given a filename being traced, return the best guess as to the source + Given a file name being traced, return the best guess as to the source file to attribute it to. """ @@ -376,18 +389,18 @@ class Coverage(object): # Jython is easy to guess. return filename[:-9] + ".py" - # No idea, just use the filename as-is. + # No idea, just use the file name as-is. return filename def _name_for_module(self, module_globals, filename): - """Get the name of the module for a set of globals and filename. + """Get the name of the module for a set of globals and file name. For configurability's sake, we allow __main__ modules to be matched by their importable name. If loaded via runpy (aka -m), we can usually recover the "original" full dotted module name, otherwise, we resort to interpreting the - filename to get the module's name. In the case that the module name + file name to get the module's name. In the case that the module name can't be determined, None is returned. """ @@ -424,7 +437,8 @@ class Coverage(object): Returns a FileDisposition object. """ - disp = FileDisposition(filename) + original_filename = filename + disp = _disposition_init(self.collector.file_disposition_class, filename) def nope(disp, reason): """Simple helper to make it easy to return NO.""" @@ -432,8 +446,8 @@ class Coverage(object): disp.reason = reason return disp - # Compiled Python files have two filenames: frame.f_code.co_filename is - # the filename at the time the .pyc was compiled. The second name is + # Compiled Python files have two file names: frame.f_code.co_filename is + # the file name at the time the .pyc was compiled. The second name is # __file__, which is where the .pyc was actually loaded from. Since # .pyc files can be moved after compilation (for example, by being # installed), we look for __file__ in the frame and prefer it to the @@ -441,31 +455,43 @@ class Coverage(object): dunder_file = frame.f_globals.get('__file__') if dunder_file: filename = self._source_for_file(dunder_file) + if original_filename and not original_filename.startswith('<'): + orig = os.path.basename(original_filename) + if orig != os.path.basename(filename): + # Files shouldn't be renamed when moved. This happens when + # exec'ing code. If it seems like something is wrong with + # the frame's file name, then just use the original. + filename = original_filename if not filename: # Empty string is pretty useless. - return nope(disp, "empty string isn't a filename") + return nope(disp, "empty string isn't a file name") if filename.startswith('memory:'): return nope(disp, "memory isn't traceable") if filename.startswith('<'): # Lots of non-file execution is represented with artificial - # filenames like "<string>", "<doctest readme.txt[0]>", or + # file names like "<string>", "<doctest readme.txt[0]>", or # "<exec_function>". Don't ever trace these executions, since we # can't do anything with the data later anyway. - return nope(disp, "not a real filename") + return nope(disp, "not a real file name") + + # pyexpat does a dumb thing, calling the trace function explicitly from + # C code with a C file name. + if re.search(r"[/\\]Modules[/\\]pyexpat.c", filename): + return nope(disp, "pyexpat lies about itself") # Jython reports the .class file to the tracer, use the source file. if filename.endswith("$py.class"): filename = filename[:-9] + ".py" - canonical = self.file_locator.canonical_filename(filename) + canonical = files.canonical_filename(filename) disp.canonical_filename = canonical # Try the plugins, see if they have an opinion about the file. plugin = None - for plugin in self.file_tracing_plugins: + for plugin in self.plugins.file_tracers: if not plugin._coverage_enabled: continue @@ -478,10 +504,9 @@ class Coverage(object): if file_tracer.has_dynamic_source_filename(): disp.has_dynamic_filename = True else: - disp.source_filename = \ - self.file_locator.canonical_filename( - file_tracer.source_filename() - ) + disp.source_filename = files.canonical_filename( + file_tracer.source_filename() + ) break except Exception: self._warn( @@ -512,7 +537,7 @@ class Coverage(object): return disp def _check_include_omit_etc_internal(self, filename, frame): - """Check a filename against the include, omit, etc, rules. + """Check a file name against the include, omit, etc, rules. Returns a string or None. String means, don't trace, and is the reason why. None means no reason found to not trace. @@ -541,8 +566,8 @@ class Coverage(object): if self.pylib_match and self.pylib_match.match(filename): return "is in the stdlib" - # We exclude the coverage code itself, since a little of it will be - # measured otherwise. + # We exclude the coverage.py code itself, since a little of it + # will be measured otherwise. if self.cover_match and self.cover_match.match(filename): return "is part of coverage.py" @@ -561,11 +586,11 @@ class Coverage(object): """ disp = self._should_trace_internal(filename, frame) if self.debug.should('trace'): - self.debug.write(disp.debug_message()) + self.debug.write(_disposition_debug_msg(disp)) return disp def _check_include_omit_etc(self, filename, frame): - """Check a filename against the include/omit/etc, rules, verbosely. + """Check a file name against the include/omit/etc, rules, verbosely. Returns a boolean: True if the file should be traced, False if not. @@ -583,33 +608,70 @@ class Coverage(object): def _warn(self, msg): """Use `msg` as a warning.""" self._warnings.append(msg) - if self.debug.should("pid"): + if self.debug.should('pid'): msg = "[%d] %s" % (os.getpid(), msg) sys.stderr.write("Coverage.py warning: %s\n" % msg) - def use_cache(self, usecache): - """Control the use of a data file (incorrectly called a cache). + def get_option(self, option_name): + """Get an option from the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with `"run:branch"`. - `usecache` is true or false, whether to read and write data on disk. + Returns the value of the option. + + .. versionadded:: 4.0 """ + return self.config.get_option(option_name) + + def set_option(self, option_name, value): + """Set an option in the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with ``"run:branch"``. + + `value` is the new value for the option. This should be a Python + value where appropriate. For example, use True for booleans, not the + string ``"True"``. + + As an example, calling:: + + cov.set_option("run:branch", True) + + has the same effect as this configuration file:: + + [run] + branch = True + + .. versionadded:: 4.0 + + """ + self.config.set_option(option_name, value) + + def use_cache(self, usecache): + """Obsolete method.""" self._init() - self.data.usefile(usecache) + if not usecache: + self._warn("use_cache(False) is no longer supported.") def load(self): """Load previously-collected coverage data from the data file.""" self._init() self.collector.reset() - self.data.read() + self.data_files.read(self.data) def start(self): """Start measuring code coverage. - Coverage measurement actually occurs in functions called after `start` - is invoked. Statements in the same scope as `start` won't be measured. + Coverage measurement actually occurs in functions called after + :meth:`start` is invoked. Statements in the same scope as + :meth:`start` won't be measured. - Once you invoke `start`, you must also call `stop` eventually, or your - process might not shut down cleanly. + Once you invoke :meth:`start`, you must also call :meth:`stop` + eventually, or your process might not shut down cleanly. """ self._init() @@ -647,6 +709,7 @@ class Coverage(object): self._init() self.collector.reset() self.data.erase() + self.data_files.erase(parallel=self.config.parallel) def clear_exclude(self, which='exclude'): """Clear the exclude list.""" @@ -688,8 +751,8 @@ class Coverage(object): def get_exclude_list(self, which='exclude'): """Return a list of excluded regex patterns. - `which` indicates which list is desired. See `exclude` for the lists - that are available, and their meaning. + `which` indicates which list is desired. See :meth:`exclude` for the + lists that are available, and their meaning. """ self._init() @@ -698,62 +761,53 @@ class Coverage(object): def save(self): """Save the collected coverage data to the data file.""" self._init() - data_suffix = self.data_suffix - if data_suffix is True: - # If data_suffix was a simple true value, then make a suffix with - # plenty of distinguishing information. We do this here in - # `save()` at the last minute so that the pid will be correct even - # if the process forks. - extra = "" - if _TEST_NAME_FILE: # pragma: debugging - with open(_TEST_NAME_FILE) as f: - test_name = f.read() - extra = "." + test_name - data_suffix = "%s%s.%s.%06d" % ( - socket.gethostname(), extra, os.getpid(), - random.randint(0, 999999) - ) + self.get_data() + self.data_files.write(self.data, suffix=self.data_suffix) - self._harvest_data() - self.data.write(suffix=data_suffix) - - def combine(self, data_dirs=None): + def combine(self, data_paths=None): """Combine together a number of similarly-named coverage data files. All coverage data files whose name starts with `data_file` (from the coverage() constructor) will be read, and combined together into the current measurements. - `data_dirs` is a list of directories from which data files should be - combined. If no list is passed, then the data files from the current - directory will be combined. + `data_paths` is a list of files or directories from which data should + be combined. If no list is passed, then the data files from the + directory indicated by the current data file (probably the current + directory) will be combined. + + .. versionadded:: 4.0 + The `data_paths` parameter. """ self._init() + self.get_data() + aliases = None if self.config.paths: - aliases = PathAliases(self.file_locator) + aliases = PathAliases() for paths in self.config.paths.values(): result = paths[0] for pattern in paths[1:]: aliases.add(pattern, result) - self.data.combine_parallel_data(aliases=aliases, data_dirs=data_dirs) - def _harvest_data(self): + self.data_files.combine_parallel_data(self.data, aliases=aliases, data_paths=data_paths) + + def get_data(self): """Get the collected data and reset the collector. Also warn about various problems collecting data. + Returns a :class:`coverage.CoverageData`, the collected coverage data. + + .. versionadded:: 4.0 + """ self._init() if not self._measured: - return + return self.data - # TODO: seems like this parallel structure is getting kinda old... - self.data.add_line_data(self.collector.get_line_data()) - self.data.add_arc_data(self.collector.get_arc_data()) - self.data.add_plugin_data(self.collector.get_plugin_data()) - self.collector.reset() + self.collector.save_data(self.data) # If there are still entries in the source_pkgs list, then we never # encountered those packages. @@ -767,20 +821,16 @@ class Coverage(object): ): self._warn("Module %s has no Python source." % pkg) else: - self._warn( - "Module %s was previously imported, " - "but not measured." % pkg - ) + self._warn("Module %s was previously imported, but not measured." % pkg) # Find out if we got any data. - summary = self.data.summary() - if not summary and self._warn_no_data: + if not self.data and self._warn_no_data: self._warn("No data was collected.") # Find files that were never executed at all. for src in self.source: for py_file in find_python_files(src): - py_file = self.file_locator.canonical_filename(py_file) + py_file = files.canonical_filename(py_file) if self.omit_match and self.omit_match.match(py_file): # Turns out this file was omitted, so don't pull it back @@ -789,7 +839,11 @@ class Coverage(object): self.data.touch_file(py_file) + if self.config.note: + self.data.add_run_info(note=self.config.note) + self._measured = False + return self.data # Backward compatibility with version 1. def analysis(self, morf): @@ -800,10 +854,10 @@ class Coverage(object): def analysis2(self, morf): """Analyze a module. - `morf` is a module or a filename. It will be analyzed to determine + `morf` is a module or a file name. It will be analyzed to determine its coverage statistics. The return value is a 5-tuple: - * The filename for the module. + * The file name for the module. * A list of line numbers of executable statements. * A list of line numbers of excluded statements. * A list of line numbers of statements not run (missing from @@ -830,19 +884,20 @@ class Coverage(object): Returns an `Analysis` object. """ - self._harvest_data() + self.get_data() if not isinstance(it, FileReporter): it = self._get_file_reporter(it) - return Analysis(self, it) + return Analysis(self.data, it) def _get_file_reporter(self, morf): - """Get a FileReporter for a module or filename.""" + """Get a FileReporter for a module or file name.""" plugin = None + file_reporter = "python" if isinstance(morf, string_class): abs_morf = abs_file(morf) - plugin_name = self.data.plugin_data().get(abs_morf) + plugin_name = self.data.file_tracer(abs_morf) if plugin_name: plugin = self.plugins.get(plugin_name) @@ -854,25 +909,19 @@ class Coverage(object): plugin._coverage_plugin_name, morf ) ) - else: - file_reporter = PythonFileReporter(morf, self) - # The FileReporter can have a name attribute, but if it doesn't, we'll - # supply it as the relative path to self.filename. - if not hasattr(file_reporter, "name"): - file_reporter.name = self.file_locator.relative_filename( - file_reporter.filename - ) + if file_reporter == "python": + file_reporter = PythonFileReporter(morf, self) return file_reporter def _get_file_reporters(self, morfs=None): - """Get a list of FileReporters for a list of modules or filenames. + """Get a list of FileReporters for a list of modules or file names. - For each module or filename in `morfs`, find a FileReporter. Return + For each module or file name in `morfs`, find a FileReporter. Return the list of FileReporters. - If `morfs` is a single module or filename, this returns a list of one + If `morfs` is a single module or file name, this returns a list of one FileReporter. If `morfs` is empty or None, then the list of all files measured is used to find the FileReporters. @@ -901,14 +950,14 @@ class Coverage(object): Each module in `morfs` is listed, with counts of statements, executed statements, missing statements, and a list of lines missed. - `include` is a list of filename patterns. Files that match will be + `include` is a list of file name patterns. Files that match will be included in the report. Files matching `omit` will not be included in the report. Returns a float, the total percentage covered. """ - self._harvest_data() + self.get_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, show_missing=show_missing, skip_covered=skip_covered, @@ -927,10 +976,10 @@ class Coverage(object): marker to indicate the coverage of the line. Covered lines have ">", excluded lines have "-", and missing lines have "!". - See `coverage.report()` for other arguments. + See :meth:`report` for other arguments. """ - self._harvest_data() + self.get_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include ) @@ -951,12 +1000,12 @@ class Coverage(object): `title` is a text string (not HTML) to use as the title of the HTML report. - See `coverage.report()` for other arguments. + See :meth:`report` for other arguments. Returns a float, the total percentage covered. """ - self._harvest_data() + self.get_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, html_dir=directory, extra_css=extra_css, html_title=title, @@ -975,12 +1024,12 @@ class Coverage(object): Each module in `morfs` is included in the report. `outfile` is the path to write the file to, "-" will write to stdout. - See `coverage.report()` for other arguments. + See :meth:`report` for other arguments. Returns a float, the total percentage covered. """ - self._harvest_data() + self.get_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, xml_output=outfile, @@ -998,10 +1047,13 @@ class Coverage(object): output_dir = os.path.dirname(self.config.xml_output) if output_dir and not os.path.isdir(output_dir): os.makedirs(output_dir) - outfile = open(self.config.xml_output, "w") + open_kwargs = {} + if env.PY3: + open_kwargs['encoding'] = 'utf8' + outfile = open(self.config.xml_output, "w", **open_kwargs) file_to_close = outfile try: - reporter = XmlReporter(self, self.config, self.file_locator) + reporter = XmlReporter(self, self.config) return reporter.report(morfs, outfile=outfile) except CoverageException: delete_file = True @@ -1018,13 +1070,9 @@ class Coverage(object): import coverage as covmod self._init() - try: - implementation = platform.python_implementation() - except AttributeError: - implementation = "unknown" ft_plugins = [] - for ft in self.file_tracing_plugins: + for ft in self.plugins.file_tracers: ft_name = ft._coverage_plugin_name if not ft._coverage_enabled: ft_name += " (disabled)" @@ -1033,16 +1081,16 @@ class Coverage(object): info = [ ('version', covmod.__version__), ('coverage', covmod.__file__), - ('cover_dir', self.cover_dir), + ('cover_dirs', self.cover_dirs), ('pylib_dirs', self.pylib_dirs), ('tracer', self.collector.tracer_name()), - ('file_tracing_plugins', ft_plugins), + ('plugins.file_tracers', ft_plugins), ('config_files', self.config.attempted_config_files), ('configs_read', self.config.config_files), - ('data_path', self.data.filename), + ('data_path', self.data_files.filename), ('python', sys.version.replace('\n', '')), ('platform', platform.platform()), - ('implementation', implementation), + ('implementation', platform.python_implementation()), ('executable', sys.executable), ('cwd', os.getcwd()), ('path', sys.path), @@ -1071,36 +1119,32 @@ class Coverage(object): return info -class FileDisposition(object): - """A simple object for noting a number of details of files to trace.""" - def __init__(self, original_filename): - self.original_filename = original_filename - self.canonical_filename = original_filename - self.source_filename = None - self.trace = False - self.reason = "" - self.file_tracer = None - self.has_dynamic_filename = False - - def __repr__(self): - ret = "FileDisposition %r" % (self.original_filename,) - if self.trace: - ret += " trace" - else: - ret += " notrace=%r" % (self.reason,) - if self.file_tracer: - ret += " file_tracer=%r" % (self.file_tracer,) - return "<" + ret + ">" - - def debug_message(self): - """Produce a debugging message explaining the outcome.""" - if self.trace: - msg = "Tracing %r" % (self.original_filename,) - if self.file_tracer: - msg += ": will be traced by %r" % self.file_tracer - else: - msg = "Not tracing %r: %s" % (self.original_filename, self.reason) - return msg +# FileDisposition "methods": FileDisposition is a pure value object, so it can +# be implemented in either C or Python. Acting on them is done with these +# functions. + +def _disposition_init(cls, original_filename): + """Construct and initialize a new FileDisposition object.""" + disp = cls() + disp.original_filename = original_filename + disp.canonical_filename = original_filename + disp.source_filename = None + disp.trace = False + disp.reason = "" + disp.file_tracer = None + disp.has_dynamic_filename = False + return disp + + +def _disposition_debug_msg(disp): + """Make a nice debug message of what the FileDisposition is doing.""" + if disp.trace: + msg = "Tracing %r" % (disp.original_filename,) + if disp.file_tracer: + msg += ": will be traced by %r" % disp.file_tracer + else: + msg = "Not tracing %r: %s" % (disp.original_filename, disp.reason) + return msg def process_startup(): @@ -1132,7 +1176,7 @@ def process_startup(): # because some virtualenv configurations make the same directory visible # twice in sys.path. This means that the .pth file will be found twice, # and executed twice, executing this function twice. We set a global - # flag (an attribute on this function) to indicate that coverage has + # flag (an attribute on this function) to indicate that coverage.py has # already been started, so we can avoid doing it twice. # # https://bitbucket.org/ned/coveragepy/issue/340/keyerror-subpy has more @@ -1140,7 +1184,7 @@ def process_startup(): if hasattr(process_startup, "done"): # We've annotated this function before, so we must have already - # started coverage in this process. Nothing to do. + # started coverage.py in this process. Nothing to do. return process_startup.done = True @@ -1148,55 +1192,3 @@ def process_startup(): cov.start() cov._warn_no_data = False cov._warn_unimported_source = False - - -# A hack for debugging testing in sub-processes. -_TEST_NAME_FILE = "" # "/tmp/covtest.txt" - - -class Plugins(object): - """The currently loaded collection of coverage.py plugins.""" - - def __init__(self): - self.order = [] - self.names = {} - - @classmethod - def load_plugins(cls, modules, config): - """Load plugins from `modules`. - - Returns a list of loaded and configured plugins. - - """ - plugins = cls() - - for module in modules: - __import__(module) - mod = sys.modules[module] - - plugin_class = getattr(mod, "Plugin", None) - if plugin_class: - options = config.get_plugin_options(module) - plugin = plugin_class(options) - plugin._coverage_plugin_name = module - plugin._coverage_enabled = True - plugins.order.append(plugin) - plugins.names[module] = plugin - else: - raise CoverageException( - "Plugin module %r didn't define a Plugin class" % module - ) - - return plugins - - def __nonzero__(self): - return bool(self.order) - - __bool__ = __nonzero__ - - def __iter__(self): - return iter(self.order) - - def get(self, plugin_name): - """Return a plugin by name.""" - return self.names[plugin_name] diff --git a/coverage/ctracer/datastack.c b/coverage/ctracer/datastack.c new file mode 100644 index 00000000..5a384e6b --- /dev/null +++ b/coverage/ctracer/datastack.c @@ -0,0 +1,42 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#include "util.h" +#include "datastack.h" + +#define STACK_DELTA 100 + +int +DataStack_init(Stats *pstats, DataStack *pdata_stack) +{ + pdata_stack->depth = -1; + pdata_stack->stack = NULL; + pdata_stack->alloc = 0; + return RET_OK; +} + +void +DataStack_dealloc(Stats *pstats, DataStack *pdata_stack) +{ + PyMem_Free(pdata_stack->stack); +} + +int +DataStack_grow(Stats *pstats, DataStack *pdata_stack) +{ + pdata_stack->depth++; + if (pdata_stack->depth >= pdata_stack->alloc) { + /* We've outgrown our data_stack array: make it bigger. */ + int bigger = pdata_stack->alloc + STACK_DELTA; + DataStackEntry * bigger_data_stack = PyMem_Realloc(pdata_stack->stack, bigger * sizeof(DataStackEntry)); + STATS( pstats->stack_reallocs++; ) + if (bigger_data_stack == NULL) { + PyErr_NoMemory(); + pdata_stack->depth--; + return RET_ERROR; + } + pdata_stack->stack = bigger_data_stack; + pdata_stack->alloc = bigger; + } + return RET_OK; +} diff --git a/coverage/ctracer/datastack.h b/coverage/ctracer/datastack.h new file mode 100644 index 00000000..78f85f7e --- /dev/null +++ b/coverage/ctracer/datastack.h @@ -0,0 +1,45 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#ifndef _COVERAGE_DATASTACK_H +#define _COVERAGE_DATASTACK_H + +#include "util.h" +#include "stats.h" + +/* An entry on the data stack. For each call frame, we need to record all + * the information needed for CTracer_handle_line to operate as quickly as + * possible. All PyObject* here are borrowed references. + */ +typedef struct DataStackEntry { + /* The current file_data dictionary. Borrowed, owned by self->data. */ + PyObject * file_data; + + /* The disposition object for this frame. If collector.py and control.py + * are working properly, this will be an instance of CFileDisposition. + */ + PyObject * disposition; + + /* The FileTracer handling this frame, or None if it's Python. */ + PyObject * file_tracer; + + /* The line number of the last line recorded, for tracing arcs. + -1 means there was no previous line, as when entering a code object. + */ + int last_line; +} DataStackEntry; + +/* A data stack is a dynamically allocated vector of DataStackEntry's. */ +typedef struct DataStack { + int depth; /* The index of the last-used entry in stack. */ + int alloc; /* number of entries allocated at stack. */ + /* The file data at each level, or NULL if not recording. */ + DataStackEntry * stack; +} DataStack; + + +int DataStack_init(Stats * pstats, DataStack *pdata_stack); +void DataStack_dealloc(Stats * pstats, DataStack *pdata_stack); +int DataStack_grow(Stats * pstats, DataStack *pdata_stack); + +#endif /* _COVERAGE_DATASTACK_H */ diff --git a/coverage/ctracer/filedisp.c b/coverage/ctracer/filedisp.c new file mode 100644 index 00000000..479a2c9f --- /dev/null +++ b/coverage/ctracer/filedisp.c @@ -0,0 +1,85 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#include "util.h" +#include "filedisp.h" + +void +CFileDisposition_dealloc(CFileDisposition *self) +{ + Py_XDECREF(self->original_filename); + Py_XDECREF(self->canonical_filename); + Py_XDECREF(self->source_filename); + Py_XDECREF(self->trace); + Py_XDECREF(self->reason); + Py_XDECREF(self->file_tracer); + Py_XDECREF(self->has_dynamic_filename); +} + +static PyMemberDef +CFileDisposition_members[] = { + { "original_filename", T_OBJECT, offsetof(CFileDisposition, original_filename), 0, + PyDoc_STR("") }, + + { "canonical_filename", T_OBJECT, offsetof(CFileDisposition, canonical_filename), 0, + PyDoc_STR("") }, + + { "source_filename", T_OBJECT, offsetof(CFileDisposition, source_filename), 0, + PyDoc_STR("") }, + + { "trace", T_OBJECT, offsetof(CFileDisposition, trace), 0, + PyDoc_STR("") }, + + { "reason", T_OBJECT, offsetof(CFileDisposition, reason), 0, + PyDoc_STR("") }, + + { "file_tracer", T_OBJECT, offsetof(CFileDisposition, file_tracer), 0, + PyDoc_STR("") }, + + { "has_dynamic_filename", T_OBJECT, offsetof(CFileDisposition, has_dynamic_filename), 0, + PyDoc_STR("") }, + + { NULL } +}; + +PyTypeObject +CFileDispositionType = { + MyType_HEAD_INIT + "coverage.CFileDispositionType", /*tp_name*/ + sizeof(CFileDisposition), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)CFileDisposition_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "CFileDisposition objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + CFileDisposition_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; diff --git a/coverage/ctracer/filedisp.h b/coverage/ctracer/filedisp.h new file mode 100644 index 00000000..ada68eaf --- /dev/null +++ b/coverage/ctracer/filedisp.h @@ -0,0 +1,26 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#ifndef _COVERAGE_FILEDISP_H +#define _COVERAGE_FILEDISP_H + +#include "util.h" +#include "structmember.h" + +typedef struct CFileDisposition { + PyObject_HEAD + + PyObject * original_filename; + PyObject * canonical_filename; + PyObject * source_filename; + PyObject * trace; + PyObject * reason; + PyObject * file_tracer; + PyObject * has_dynamic_filename; +} CFileDisposition; + +void CFileDisposition_dealloc(CFileDisposition *self); + +extern PyTypeObject CFileDispositionType; + +#endif /* _COVERAGE_FILEDISP_H */ diff --git a/coverage/ctracer/module.c b/coverage/ctracer/module.c new file mode 100644 index 00000000..76231859 --- /dev/null +++ b/coverage/ctracer/module.c @@ -0,0 +1,108 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#include "util.h" +#include "tracer.h" +#include "filedisp.h" + +/* Module definition */ + +#define MODULE_DOC PyDoc_STR("Fast coverage tracer.") + +#if PY_MAJOR_VERSION >= 3 + +static PyModuleDef +moduledef = { + PyModuleDef_HEAD_INIT, + "coverage.tracer", + MODULE_DOC, + -1, + NULL, /* methods */ + NULL, + NULL, /* traverse */ + NULL, /* clear */ + NULL +}; + + +PyObject * +PyInit_tracer(void) +{ + PyObject * mod = PyModule_Create(&moduledef); + if (mod == NULL) { + return NULL; + } + + if (CTracer_intern_strings() < 0) { + return NULL; + } + + /* Initialize CTracer */ + CTracerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&CTracerType) < 0) { + Py_DECREF(mod); + return NULL; + } + + Py_INCREF(&CTracerType); + if (PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType) < 0) { + Py_DECREF(mod); + Py_DECREF(&CTracerType); + return NULL; + } + + /* Initialize CFileDisposition */ + CFileDispositionType.tp_new = PyType_GenericNew; + if (PyType_Ready(&CFileDispositionType) < 0) { + Py_DECREF(mod); + Py_DECREF(&CTracerType); + return NULL; + } + + Py_INCREF(&CFileDispositionType); + if (PyModule_AddObject(mod, "CFileDisposition", (PyObject *)&CFileDispositionType) < 0) { + Py_DECREF(mod); + Py_DECREF(&CTracerType); + Py_DECREF(&CFileDispositionType); + return NULL; + } + + return mod; +} + +#else + +void +inittracer(void) +{ + PyObject * mod; + + mod = Py_InitModule3("coverage.tracer", NULL, MODULE_DOC); + if (mod == NULL) { + return; + } + + if (CTracer_intern_strings() < 0) { + return; + } + + /* Initialize CTracer */ + CTracerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&CTracerType) < 0) { + return; + } + + Py_INCREF(&CTracerType); + PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType); + + /* Initialize CFileDisposition */ + CFileDispositionType.tp_new = PyType_GenericNew; + if (PyType_Ready(&CFileDispositionType) < 0) { + return; + } + + Py_INCREF(&CFileDispositionType); + PyModule_AddObject(mod, "CFileDisposition", (PyObject *)&CFileDispositionType); +} + +#endif /* Py3k */ diff --git a/coverage/ctracer/stats.h b/coverage/ctracer/stats.h new file mode 100644 index 00000000..ceba79bd --- /dev/null +++ b/coverage/ctracer/stats.h @@ -0,0 +1,30 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#ifndef _COVERAGE_STATS_H +#define _COVERAGE_STATS_H + +#include "util.h" + +#if COLLECT_STATS +#define STATS(x) x +#else +#define STATS(x) +#endif + +typedef struct Stats { + unsigned int calls; /* Need at least one member, but the rest only if needed. */ +#if COLLECT_STATS + unsigned int lines; + unsigned int returns; + unsigned int exceptions; + unsigned int others; + unsigned int new_files; + unsigned int missed_returns; + unsigned int stack_reallocs; + unsigned int errors; + unsigned int pycalls; +#endif +} Stats; + +#endif /* _COVERAGE_STATS_H */ diff --git a/coverage/tracer.c b/coverage/ctracer/tracer.c index fe40fc67..25036f99 100644 --- a/coverage/tracer.c +++ b/coverage/ctracer/tracer.c @@ -1,52 +1,12 @@ -/* C-based Tracer for Coverage. */ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ -#include "Python.h" -#include "structmember.h" -#include "frameobject.h" -#include "opcode.h" +/* C-based Tracer for coverage.py. */ -/* Compile-time debugging helpers */ -#undef WHAT_LOG /* Define to log the WHAT params in the trace function. */ -#undef TRACE_LOG /* Define to log our bookkeeping. */ -#undef COLLECT_STATS /* Collect counters: stats are printed when tracer is stopped. */ - -#if COLLECT_STATS -#define STATS(x) x -#else -#define STATS(x) -#endif - -/* Py 2.x and 3.x compatibility */ - -#if PY_MAJOR_VERSION >= 3 - -#define MyText_Type PyUnicode_Type -#define MyText_AS_BYTES(o) PyUnicode_AsASCIIString(o) -#define MyBytes_AS_STRING(o) PyBytes_AS_STRING(o) -#define MyText_AsString(o) PyUnicode_AsUTF8(o) -#define MyText_FromFormat PyUnicode_FromFormat -#define MyInt_FromInt(i) PyLong_FromLong((long)i) -#define MyInt_AsInt(o) (int)PyLong_AsLong(o) - -#define MyType_HEAD_INIT PyVarObject_HEAD_INIT(NULL, 0) - -#else - -#define MyText_Type PyString_Type -#define MyText_AS_BYTES(o) (Py_INCREF(o), o) -#define MyBytes_AS_STRING(o) PyString_AS_STRING(o) -#define MyText_AsString(o) PyString_AsString(o) -#define MyText_FromFormat PyUnicode_FromFormat -#define MyInt_FromInt(i) PyInt_FromLong((long)i) -#define MyInt_AsInt(o) (int)PyInt_AsLong(o) - -#define MyType_HEAD_INIT PyObject_HEAD_INIT(NULL) 0, - -#endif /* Py3k */ - -/* The values returned to indicate ok or error. */ -#define RET_OK 0 -#define RET_ERROR -1 +#include "util.h" +#include "datastack.h" +#include "filedisp.h" +#include "tracer.h" /* Python C API helpers. */ @@ -63,155 +23,49 @@ pyint_as_int(PyObject * pyint, int *pint) } -/* An entry on the data stack. For each call frame, we need to record all - * the information needed for CTracer_handle_line to operate as quickly as - * possible. - */ -typedef struct { - /* The current file_data dictionary. Borrowed, owned by self->data. */ - PyObject * file_data; - - /* The disposition object for this frame. */ - PyObject * disposition; - - /* The FileTracer handling this frame, or None if it's Python. */ - PyObject * file_tracer; - - /* The line number of the last line recorded, for tracing arcs. - -1 means there was no previous line, as when entering a code object. - */ - int last_line; -} DataStackEntry; - -/* A data stack is a dynamically allocated vector of DataStackEntry's. */ -typedef struct { - int depth; /* The index of the last-used entry in stack. */ - int alloc; /* number of entries allocated at stack. */ - /* The file data at each level, or NULL if not recording. */ - DataStackEntry * stack; -} DataStack; - -/* The CTracer type. */ - -typedef struct { - PyObject_HEAD - - /* Python objects manipulated directly by the Collector class. */ - PyObject * should_trace; - PyObject * check_include; - PyObject * warn; - PyObject * concur_id_func; - PyObject * data; - PyObject * plugin_data; - PyObject * should_trace_cache; - PyObject * arcs; - - /* Has the tracer been started? */ - int started; - /* Are we tracing arcs, or just lines? */ - int tracing_arcs; - - /* - The data stack is a stack of dictionaries. Each dictionary collects - data for a single source file. The data stack parallels the call stack: - each call pushes the new frame's file data onto the data stack, and each - return pops file data off. - - The file data is a dictionary whose form depends on the tracing options. - If tracing arcs, the keys are line number pairs. If not tracing arcs, - the keys are line numbers. In both cases, the value is irrelevant - (None). - */ - - DataStack data_stack; /* Used if we aren't doing concurrency. */ - - PyObject * data_stack_index; /* Used if we are doing concurrency. */ - DataStack * data_stacks; - int data_stacks_alloc; - int data_stacks_used; - DataStack * pdata_stack; - - /* The current file's data stack entry, copied from the stack. */ - DataStackEntry cur_entry; +/* Interned strings to speed GetAttr etc. */ - /* The parent frame for the last exception event, to fix missing returns. */ - PyFrameObject * last_exc_back; - int last_exc_firstlineno; +static PyObject *str_trace; +static PyObject *str_file_tracer; +static PyObject *str__coverage_enabled; +static PyObject *str__coverage_plugin; +static PyObject *str__coverage_plugin_name; +static PyObject *str_dynamic_source_filename; +static PyObject *str_line_number_range; -#if COLLECT_STATS - struct { - unsigned int calls; - unsigned int lines; - unsigned int returns; - unsigned int exceptions; - unsigned int others; - unsigned int new_files; - unsigned int missed_returns; - unsigned int stack_reallocs; - unsigned int errors; - } stats; -#endif /* COLLECT_STATS */ -} CTracer; +int +CTracer_intern_strings(void) +{ + int ret = RET_ERROR; +#define INTERN_STRING(v, s) \ + v = MyText_InternFromString(s); \ + if (v == NULL) { \ + goto error; \ + } -#define STACK_DELTA 100 + INTERN_STRING(str_trace, "trace") + INTERN_STRING(str_file_tracer, "file_tracer") + INTERN_STRING(str__coverage_enabled, "_coverage_enabled") + INTERN_STRING(str__coverage_plugin, "_coverage_plugin") + INTERN_STRING(str__coverage_plugin_name, "_coverage_plugin_name") + INTERN_STRING(str_dynamic_source_filename, "dynamic_source_filename") + INTERN_STRING(str_line_number_range, "line_number_range") -static int -DataStack_init(CTracer *self, DataStack *pdata_stack) -{ - pdata_stack->depth = -1; - pdata_stack->stack = NULL; - pdata_stack->alloc = 0; - return RET_OK; -} - -static void -DataStack_dealloc(CTracer *self, DataStack *pdata_stack) -{ - PyMem_Free(pdata_stack->stack); -} + ret = RET_OK; -static int -DataStack_grow(CTracer *self, DataStack *pdata_stack) -{ - pdata_stack->depth++; - if (pdata_stack->depth >= pdata_stack->alloc) { - STATS( self->stats.stack_reallocs++; ) - /* We've outgrown our data_stack array: make it bigger. */ - int bigger = pdata_stack->alloc + STACK_DELTA; - DataStackEntry * bigger_data_stack = PyMem_Realloc(pdata_stack->stack, bigger * sizeof(DataStackEntry)); - if (bigger_data_stack == NULL) { - PyErr_NoMemory(); - pdata_stack->depth--; - return RET_ERROR; - } - pdata_stack->stack = bigger_data_stack; - pdata_stack->alloc = bigger; - } - return RET_OK; +error: + return ret; } - static void CTracer_disable_plugin(CTracer *self, PyObject * disposition); static int CTracer_init(CTracer *self, PyObject *args_unused, PyObject *kwds_unused) { int ret = RET_ERROR; - PyObject * weakref = NULL; - if (DataStack_init(self, &self->data_stack) < 0) { - goto error; - } - - weakref = PyImport_ImportModule("weakref"); - if (weakref == NULL) { - goto error; - } - self->data_stack_index = PyObject_CallMethod(weakref, "WeakKeyDictionary", NULL); - Py_XDECREF(weakref); - - if (self->data_stack_index == NULL) { + if (DataStack_init(&self->stats, &self->data_stack) < 0) { goto error; } @@ -243,13 +97,13 @@ CTracer_dealloc(CTracer *self) Py_XDECREF(self->warn); Py_XDECREF(self->concur_id_func); Py_XDECREF(self->data); - Py_XDECREF(self->plugin_data); + Py_XDECREF(self->file_tracers); Py_XDECREF(self->should_trace_cache); - DataStack_dealloc(self, &self->data_stack); + DataStack_dealloc(&self->stats, &self->data_stack); if (self->data_stacks) { for (i = 0; i < self->data_stacks_used; i++) { - DataStack_dealloc(self, self->data_stacks + i); + DataStack_dealloc(&self->stats, self->data_stacks + i); } PyMem_Free(self->data_stacks); } @@ -345,6 +199,23 @@ CTracer_set_pdata_stack(CTracer *self) if (self->concur_id_func != Py_None) { int the_index = 0; + if (self->data_stack_index == NULL) { + PyObject * weakref = NULL; + + weakref = PyImport_ImportModule("weakref"); + if (weakref == NULL) { + goto error; + } + STATS( self->stats.pycalls++; ) + self->data_stack_index = PyObject_CallMethod(weakref, "WeakKeyDictionary", NULL); + Py_XDECREF(weakref); + + if (self->data_stack_index == NULL) { + goto error; + } + } + + STATS( self->stats.pycalls++; ) co_obj = PyObject_CallObject(self->concur_id_func, NULL); if (co_obj == NULL) { goto error; @@ -374,7 +245,7 @@ CTracer_set_pdata_stack(CTracer *self) self->data_stacks = bigger_stacks; self->data_stacks_alloc = bigger; } - DataStack_init(self, &self->data_stacks[the_index]); + DataStack_init(&self->stats, &self->data_stacks[the_index]); } else { if (pyint_as_int(stack_index, &the_index) < 0) { @@ -451,16 +322,19 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) int ret2; /* Owned references that we clean up at the very end of the function. */ - PyObject * tracename = NULL; PyObject * disposition = NULL; - PyObject * disp_trace = NULL; - PyObject * file_tracer = NULL; PyObject * plugin = NULL; PyObject * plugin_name = NULL; - PyObject * has_dynamic_filename = NULL; + PyObject * next_tracename = NULL; /* Borrowed references. */ PyObject * filename = NULL; + PyObject * disp_trace = NULL; + PyObject * tracename = NULL; + PyObject * file_tracer = NULL; + PyObject * has_dynamic_filename = NULL; + + CFileDisposition * pdisp = NULL; STATS( self->stats.calls++; ) @@ -468,7 +342,7 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) if (CTracer_set_pdata_stack(self) < 0) { goto error; } - if (DataStack_grow(self, self->pdata_stack) < 0) { + if (DataStack_grow(&self->stats, self->pdata_stack) < 0) { goto error; } @@ -483,8 +357,10 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) goto error; } STATS( self->stats.new_files++; ) + /* We've never considered this file before. */ /* Ask should_trace about it. */ + STATS( self->stats.pycalls++; ) disposition = PyObject_CallFunctionObjArgs(self->should_trace, filename, frame, NULL); if (disposition == NULL) { /* An error occurred inside should_trace. */ @@ -498,40 +374,48 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) Py_INCREF(disposition); } - disp_trace = PyObject_GetAttrString(disposition, "trace"); - if (disp_trace == NULL) { - goto error; + if (disposition == Py_None) { + /* A later check_include returned false, so don't trace it. */ + disp_trace = Py_False; + } + else { + /* The object we got is a CFileDisposition, use it efficiently. */ + pdisp = (CFileDisposition *) disposition; + disp_trace = pdisp->trace; + if (disp_trace == NULL) { + goto error; + } } if (disp_trace == Py_True) { /* If tracename is a string, then we're supposed to trace. */ - tracename = PyObject_GetAttrString(disposition, "source_filename"); + tracename = pdisp->source_filename; if (tracename == NULL) { goto error; } - file_tracer = PyObject_GetAttrString(disposition, "file_tracer"); + file_tracer = pdisp->file_tracer; if (file_tracer == NULL) { goto error; } if (file_tracer != Py_None) { - plugin = PyObject_GetAttrString(file_tracer, "_coverage_plugin"); + plugin = PyObject_GetAttr(file_tracer, str__coverage_plugin); if (plugin == NULL) { goto error; } - plugin_name = PyObject_GetAttrString(plugin, "_coverage_plugin_name"); + plugin_name = PyObject_GetAttr(plugin, str__coverage_plugin_name); if (plugin_name == NULL) { goto error; } } - has_dynamic_filename = PyObject_GetAttrString(disposition, "has_dynamic_filename"); + has_dynamic_filename = pdisp->has_dynamic_filename; if (has_dynamic_filename == NULL) { goto error; } if (has_dynamic_filename == Py_True) { - PyObject * next_tracename = NULL; - next_tracename = PyObject_CallMethod( - file_tracer, "dynamic_source_filename", - "OO", tracename, frame + STATS( self->stats.pycalls++; ) + next_tracename = PyObject_CallMethodObjArgs( + file_tracer, str_dynamic_source_filename, + tracename, frame, NULL ); if (next_tracename == NULL) { /* An exception from the function. Alert the user with a @@ -541,37 +425,41 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) /* Because we handled the error, goto ok. */ goto ok; } - Py_DECREF(tracename); tracename = next_tracename; if (tracename != Py_None) { /* Check the dynamic source filename against the include rules. */ PyObject * included = NULL; + int should_include; included = PyDict_GetItem(self->should_trace_cache, tracename); if (included == NULL) { + PyObject * should_include_bool; if (PyErr_Occurred()) { goto error; } STATS( self->stats.new_files++; ) - included = PyObject_CallFunctionObjArgs(self->check_include, tracename, frame, NULL); - if (included == NULL) { + STATS( self->stats.pycalls++; ) + should_include_bool = PyObject_CallFunctionObjArgs(self->check_include, tracename, frame, NULL); + if (should_include_bool == NULL) { goto error; } - if (PyDict_SetItem(self->should_trace_cache, tracename, included) < 0) { + should_include = (should_include_bool == Py_True); + Py_DECREF(should_include_bool); + if (PyDict_SetItem(self->should_trace_cache, tracename, should_include ? disposition : Py_None) < 0) { goto error; } } - if (included != Py_True) { - Py_DECREF(tracename); + else { + should_include = (included != Py_None); + } + if (!should_include) { tracename = Py_None; - Py_INCREF(tracename); } } } } else { tracename = Py_None; - Py_INCREF(tracename); } if (tracename != Py_None) { @@ -593,7 +481,7 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame) /* If the disposition mentions a plugin, record that. */ if (file_tracer != Py_None) { - ret2 = PyDict_SetItem(self->plugin_data, tracename, plugin_name); + ret2 = PyDict_SetItem(self->file_tracers, tracename, plugin_name); if (ret2 < 0) { goto error; } @@ -626,13 +514,10 @@ ok: ret = RET_OK; error: - Py_XDECREF(tracename); + Py_XDECREF(next_tracename); Py_XDECREF(disposition); - Py_XDECREF(disp_trace); - Py_XDECREF(file_tracer); Py_XDECREF(plugin); Py_XDECREF(plugin_name); - Py_XDECREF(has_dynamic_filename); return ret; } @@ -647,7 +532,9 @@ CTracer_disable_plugin(CTracer *self, PyObject * disposition) PyObject * msg = NULL; PyObject * ignored = NULL; - file_tracer = PyObject_GetAttrString(disposition, "file_tracer"); + PyErr_Print(); + + file_tracer = PyObject_GetAttr(disposition, str_file_tracer); if (file_tracer == NULL) { goto error; } @@ -655,33 +542,32 @@ CTracer_disable_plugin(CTracer *self, PyObject * disposition) /* This shouldn't happen... */ goto ok; } - plugin = PyObject_GetAttrString(file_tracer, "_coverage_plugin"); + plugin = PyObject_GetAttr(file_tracer, str__coverage_plugin); if (plugin == NULL) { goto error; } - plugin_name = PyObject_GetAttrString(plugin, "_coverage_plugin_name"); + plugin_name = PyObject_GetAttr(plugin, str__coverage_plugin_name); if (plugin_name == NULL) { goto error; } msg = MyText_FromFormat( - "Disabling plugin '%s' due to an exception:", + "Disabling plugin '%s' due to previous exception", MyText_AsString(plugin_name) ); if (msg == NULL) { goto error; } + STATS( self->stats.pycalls++; ) ignored = PyObject_CallFunctionObjArgs(self->warn, msg, NULL); if (ignored == NULL) { goto error; } - PyErr_Print(); - /* Disable the plugin for future files, and stop tracing this file. */ - if (PyObject_SetAttrString(plugin, "_coverage_enabled", Py_False) < 0) { + if (PyObject_SetAttr(plugin, str__coverage_enabled, Py_False) < 0) { goto error; } - if (PyObject_SetAttrString(disposition, "trace", Py_False) < 0) { + if (PyObject_SetAttr(disposition, str_trace, Py_False) < 0) { goto error; } @@ -746,12 +632,14 @@ CTracer_handle_line(CTracer *self, PyFrameObject *frame) if (self->pdata_stack->depth >= 0) { SHOWLOG(self->pdata_stack->depth, frame->f_lineno, frame->f_code->co_filename, "line"); if (self->cur_entry.file_data) { - int lineno_from, lineno_to; + int lineno_from = -1; + int lineno_to = -1; /* We're tracing in this frame: record something. */ if (self->cur_entry.file_tracer != Py_None) { PyObject * from_to = NULL; - from_to = PyObject_CallMethod(self->cur_entry.file_tracer, "line_number_range", "O", frame); + STATS( self->stats.pycalls++; ) + from_to = PyObject_CallMethodObjArgs(self->cur_entry.file_tracer, str_line_number_range, frame, NULL); if (from_to == NULL) { goto error; } @@ -814,8 +702,18 @@ CTracer_handle_return(CTracer *self, PyFrameObject *frame) } if (self->pdata_stack->depth >= 0) { if (self->tracing_arcs && self->cur_entry.file_data) { - /* Need to distinguish between RETURN_VALUE and YIELD_VALUE. */ - int bytecode = MyBytes_AS_STRING(frame->f_code->co_code)[frame->f_lasti]; + /* Need to distinguish between RETURN_VALUE and YIELD_VALUE. Read + * the current bytecode to see what it is. In unusual circumstances + * (Cython code), co_code can be the empty string, so range-check + * f_lasti before reading the byte. + */ + int bytecode = RETURN_VALUE; + PyObject * pCode = frame->f_code->co_code; + int lasti = frame->f_lasti; + + if (lasti < MyBytes_GET_SIZE(pCode)) { + bytecode = MyBytes_AS_STRING(pCode)[lasti]; + } if (bytecode != YIELD_VALUE) { int first = frame->f_code->co_firstlineno; if (CTracer_record_pair(self, self->cur_entry.last_line, -first) < 0) { @@ -873,7 +771,7 @@ CTracer_trace(CTracer *self, PyFrameObject *frame, int what, PyObject *arg_unuse #endif #if WHAT_LOG - if (what <= sizeof(what_sym)/sizeof(const char *)) { + if (what <= (int)(sizeof(what_sym)/sizeof(const char *))) { ascii = MyText_AS_BYTES(frame->f_code->co_filename); printf("trace: %s @ %s %d\n", what_sym[what], MyBytes_AS_STRING(ascii), frame->f_lineno); Py_DECREF(ascii); @@ -964,6 +862,7 @@ CTracer_call(CTracer *self, PyObject *args, PyObject *kwds) int what; int orig_lineno; PyObject *ret = NULL; + PyObject * ascii = NULL; static char *what_names[] = { "call", "exception", "line", "return", @@ -971,10 +870,6 @@ CTracer_call(CTracer *self, PyObject *args, PyObject *kwds) NULL }; - #if WHAT_LOG - printf("pytrace\n"); - #endif - static char *kwlist[] = {"frame", "event", "arg", "lineno", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!O|i:Tracer_call", kwlist, @@ -985,14 +880,21 @@ CTracer_call(CTracer *self, PyObject *args, PyObject *kwds) /* In Python, the what argument is a string, we need to find an int for the C function. */ for (what = 0; what_names[what]; what++) { - PyObject *ascii = MyText_AS_BYTES(what_str); - int should_break = !strcmp(MyBytes_AS_STRING(ascii), what_names[what]); + int should_break; + ascii = MyText_AS_BYTES(what_str); + should_break = !strcmp(MyBytes_AS_STRING(ascii), what_names[what]); Py_DECREF(ascii); if (should_break) { break; } } + #if WHAT_LOG + ascii = MyText_AS_BYTES(frame->f_code->co_filename); + printf("pytrace: %s @ %s %d\n", what_sym[what], MyBytes_AS_STRING(ascii), frame->f_lineno); + Py_DECREF(ascii); + #endif + /* Save off the frame's lineno, and use the forced one, if provided. */ orig_lineno = frame->f_lineno; if (lineno > 0) { @@ -1008,6 +910,32 @@ CTracer_call(CTracer *self, PyObject *args, PyObject *kwds) /* Clean up. */ frame->f_lineno = orig_lineno; + /* For better speed, install ourselves the C way so that future calls go + directly to CTracer_trace, without this intermediate function. + + Only do this if this is a CALL event, since new trace functions only + take effect then. If we don't condition it on CALL, then we'll clobber + the new trace function before it has a chance to get called. To + understand why, there are three internal values to track: frame.f_trace, + c_tracefunc, and c_traceobj. They are explained here: + http://nedbatchelder.com/text/trace-function.html + + Without the conditional on PyTrace_CALL, this is what happens: + + def func(): # f_trace c_tracefunc c_traceobj + # -------------- -------------- -------------- + # CTracer CTracer.trace CTracer + sys.settrace(my_func) + # CTracer trampoline my_func + # Now Python calls trampoline(CTracer), which calls this function + # which calls PyEval_SetTrace below, setting us as the tracer again: + # CTracer CTracer.trace CTracer + # and it's as if the settrace never happened. + */ + if (what == PyTrace_CALL) { + PyEval_SetTrace((Py_tracefunc)CTracer_trace, (PyObject*)self); + } + done: return ret; } @@ -1017,7 +945,7 @@ CTracer_start(CTracer *self, PyObject *args_unused) { PyEval_SetTrace((Py_tracefunc)CTracer_trace, (PyObject*)self); self->started = 1; - self->tracing_arcs = self->arcs && PyObject_IsTrue(self->arcs); + self->tracing_arcs = self->trace_arcs && PyObject_IsTrue(self->trace_arcs); self->cur_entry.last_line = -1; /* start() returns a trace function usable with sys.settrace() */ @@ -1041,7 +969,7 @@ CTracer_get_stats(CTracer *self) { #if COLLECT_STATS return Py_BuildValue( - "{sI,sI,sI,sI,sI,sI,sI,sI,si,sI}", + "{sI,sI,sI,sI,sI,sI,sI,sI,si,sI,sI}", "calls", self->stats.calls, "lines", self->stats.lines, "returns", self->stats.returns, @@ -1051,7 +979,8 @@ CTracer_get_stats(CTracer *self) "missed_returns", self->stats.missed_returns, "stack_reallocs", self->stats.stack_reallocs, "stack_alloc", self->pdata_stack->alloc, - "errors", self->stats.errors + "errors", self->stats.errors, + "pycalls", self->stats.pycalls ); #else Py_RETURN_NONE; @@ -1075,13 +1004,13 @@ CTracer_members[] = { { "data", T_OBJECT, offsetof(CTracer, data), 0, PyDoc_STR("The raw dictionary of trace data.") }, - { "plugin_data", T_OBJECT, offsetof(CTracer, plugin_data), 0, - PyDoc_STR("Mapping from filename to plugin name.") }, + { "file_tracers", T_OBJECT, offsetof(CTracer, file_tracers), 0, + PyDoc_STR("Mapping from file name to plugin name.") }, { "should_trace_cache", T_OBJECT, offsetof(CTracer, should_trace_cache), 0, PyDoc_STR("Dictionary caching should_trace results.") }, - { "arcs", T_OBJECT, offsetof(CTracer, arcs), 0, + { "trace_arcs", T_OBJECT, offsetof(CTracer, trace_arcs), 0, PyDoc_STR("Should we trace arcs, or just lines?") }, { NULL } @@ -1101,7 +1030,7 @@ CTracer_methods[] = { { NULL } }; -static PyTypeObject +PyTypeObject CTracerType = { MyType_HEAD_INIT "coverage.CTracer", /*tp_name*/ @@ -1142,70 +1071,3 @@ CTracerType = { 0, /* tp_alloc */ 0, /* tp_new */ }; - -/* Module definition */ - -#define MODULE_DOC PyDoc_STR("Fast coverage tracer.") - -#if PY_MAJOR_VERSION >= 3 - -static PyModuleDef -moduledef = { - PyModuleDef_HEAD_INIT, - "coverage.tracer", - MODULE_DOC, - -1, - NULL, /* methods */ - NULL, - NULL, /* traverse */ - NULL, /* clear */ - NULL -}; - - -PyObject * -PyInit_tracer(void) -{ - PyObject * mod = PyModule_Create(&moduledef); - if (mod == NULL) { - return NULL; - } - - CTracerType.tp_new = PyType_GenericNew; - if (PyType_Ready(&CTracerType) < 0) { - Py_DECREF(mod); - return NULL; - } - - Py_INCREF(&CTracerType); - if (PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType) < 0) { - Py_DECREF(mod); - Py_DECREF(&CTracerType); - return NULL; - } - - return mod; -} - -#else - -void -inittracer(void) -{ - PyObject * mod; - - mod = Py_InitModule3("coverage.tracer", NULL, MODULE_DOC); - if (mod == NULL) { - return; - } - - CTracerType.tp_new = PyType_GenericNew; - if (PyType_Ready(&CTracerType) < 0) { - return; - } - - Py_INCREF(&CTracerType); - PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType); -} - -#endif /* Py3k */ diff --git a/coverage/ctracer/tracer.h b/coverage/ctracer/tracer.h new file mode 100644 index 00000000..053fbf62 --- /dev/null +++ b/coverage/ctracer/tracer.h @@ -0,0 +1,68 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#ifndef _COVERAGE_TRACER_H +#define _COVERAGE_TRACER_H + +#include "util.h" +#include "structmember.h" +#include "frameobject.h" +#include "opcode.h" + +#include "datastack.h" + +/* The CTracer type. */ + +typedef struct CTracer { + PyObject_HEAD + + /* Python objects manipulated directly by the Collector class. */ + PyObject * should_trace; + PyObject * check_include; + PyObject * warn; + PyObject * concur_id_func; + PyObject * data; + PyObject * file_tracers; + PyObject * should_trace_cache; + PyObject * trace_arcs; + + /* Has the tracer been started? */ + int started; + /* Are we tracing arcs, or just lines? */ + int tracing_arcs; + + /* + The data stack is a stack of dictionaries. Each dictionary collects + data for a single source file. The data stack parallels the call stack: + each call pushes the new frame's file data onto the data stack, and each + return pops file data off. + + The file data is a dictionary whose form depends on the tracing options. + If tracing arcs, the keys are line number pairs. If not tracing arcs, + the keys are line numbers. In both cases, the value is irrelevant + (None). + */ + + DataStack data_stack; /* Used if we aren't doing concurrency. */ + + PyObject * data_stack_index; /* Used if we are doing concurrency. */ + DataStack * data_stacks; + int data_stacks_alloc; + int data_stacks_used; + DataStack * pdata_stack; + + /* The current file's data stack entry, copied from the stack. */ + DataStackEntry cur_entry; + + /* The parent frame for the last exception event, to fix missing returns. */ + PyFrameObject * last_exc_back; + int last_exc_firstlineno; + + Stats stats; +} CTracer; + +int CTracer_intern_strings(void); + +extern PyTypeObject CTracerType; + +#endif /* _COVERAGE_TRACER_H */ diff --git a/coverage/ctracer/util.h b/coverage/ctracer/util.h new file mode 100644 index 00000000..bb3ad5a3 --- /dev/null +++ b/coverage/ctracer/util.h @@ -0,0 +1,52 @@ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +#ifndef _COVERAGE_UTIL_H +#define _COVERAGE_UTIL_H + +#include <Python.h> + +/* Compile-time debugging helpers */ +#undef WHAT_LOG /* Define to log the WHAT params in the trace function. */ +#undef TRACE_LOG /* Define to log our bookkeeping. */ +#undef COLLECT_STATS /* Collect counters: stats are printed when tracer is stopped. */ + +/* Py 2.x and 3.x compatibility */ + +#if PY_MAJOR_VERSION >= 3 + +#define MyText_Type PyUnicode_Type +#define MyText_AS_BYTES(o) PyUnicode_AsASCIIString(o) +#define MyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) +#define MyBytes_AS_STRING(o) PyBytes_AS_STRING(o) +#define MyText_AsString(o) PyUnicode_AsUTF8(o) +#define MyText_FromFormat PyUnicode_FromFormat +#define MyInt_FromInt(i) PyLong_FromLong((long)i) +#define MyInt_AsInt(o) (int)PyLong_AsLong(o) +#define MyText_InternFromString(s) \ + PyUnicode_InternFromString(s) + +#define MyType_HEAD_INIT PyVarObject_HEAD_INIT(NULL, 0) + +#else + +#define MyText_Type PyString_Type +#define MyText_AS_BYTES(o) (Py_INCREF(o), o) +#define MyBytes_GET_SIZE(o) PyString_GET_SIZE(o) +#define MyBytes_AS_STRING(o) PyString_AS_STRING(o) +#define MyText_AsString(o) PyString_AsString(o) +#define MyText_FromFormat PyUnicode_FromFormat +#define MyInt_FromInt(i) PyInt_FromLong((long)i) +#define MyInt_AsInt(o) (int)PyInt_AsLong(o) +#define MyText_InternFromString(s) \ + PyString_InternFromString(s) + +#define MyType_HEAD_INIT PyObject_HEAD_INIT(NULL) 0, + +#endif /* Py3k */ + +/* The values returned to indicate ok or error. */ +#define RET_OK 0 +#define RET_ERROR -1 + +#endif /* _COVERAGE_UTIL_H */ diff --git a/coverage/data.py b/coverage/data.py index 8a699b5b..17cf73ce 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -1,305 +1,766 @@ -"""Coverage data for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Coverage data for coverage.py.""" import glob +import json +import optparse import os - -from coverage.backward import iitems, pickle +import os.path +import random +import re +import socket + +from coverage import env +from coverage.backward import iitems, string_class +from coverage.debug import _TEST_NAME_FILE from coverage.files import PathAliases -from coverage.misc import file_be_gone +from coverage.misc import CoverageException, file_be_gone, isolate_module + +os = isolate_module(os) class CoverageData(object): """Manages collected coverage data, including file storage. - The data file format is a pickled dict, with these keys: + This class is the public supported API to the data coverage.py collects + during program execution. It includes information about what code was + executed. It does not include information from the analysis phase, to + determine what lines could have been executed, or what lines were not + executed. - * collector: a string identifying the collecting software + .. note:: - * lines: a dict mapping filenames to sorted lists of line numbers - executed: - { 'file1': [17,23,45], 'file2': [1,2,3], ... } + The file format is not documented or guaranteed. It will change in + the future, in possibly complicated ways. Do not read coverage.py + data files directly. Use this API to avoid disruption. - * arcs: a dict mapping filenames to sorted lists of line number pairs: - { 'file1': [(17,23), (17,25), (25,26)], ... } + There are a number of kinds of data that can be collected: - * plugins: a dict mapping filenames to plugin names: - { 'file1': "django.coverage", ... } - # TODO: how to handle the difference between a plugin module - # name, and the class in the module? + * **lines**: the line numbers of source lines that were executed. + These are always available. - """ + * **arcs**: pairs of source and destination line numbers for transitions + between source lines. These are only available if branch coverage was + used. - def __init__(self, basename=None, collector=None, debug=None): - """Create a CoverageData. + * **file tracer names**: the module names of the file tracer plugins that + handled each file in the data. - `basename` is the name of the file to use for storing data. + * **run information**: information about the program execution. This is + written during "coverage run", and then accumulated during "coverage + combine". - `collector` is a string describing the coverage measurement software. + Lines, arcs, and file tracer names are stored for each source file. File + names in this API are case-sensitive, even on platforms with + case-insensitive file systems. - `debug` is a `DebugControl` object for writing debug messages. + To read a coverage.py data file, use :meth:`read_file`, or + :meth:`read_fileobj` if you have an already-opened file. You can then + access the line, arc, or file tracer data with :meth:`lines`, :meth:`arcs`, + or :meth:`file_tracer`. Run information is available with + :meth:`run_infos`. + + The :meth:`has_arcs` method indicates whether arc data is available. You + can get a list of the files in the data with :meth:`measured_files`. + A summary of the line data is available from :meth:`line_counts`. As with + most Python containers, you can determine if there is any data at all by + using this object as a boolean value. - """ - self.collector = collector or 'unknown' - self.debug = debug - self.use_file = True + Most data files will be created by coverage.py itself, but you can use + methods here to create data files if you like. The :meth:`add_lines`, + :meth:`add_arcs`, and :meth:`add_file_tracers` methods add data, in ways + that are convenient for coverage.py. The :meth:`add_run_info` method adds + key-value pairs to the run information. - # Construct the filename that will be used for data file storage, if we - # ever do any file storage. - self.filename = basename or ".coverage" - self.filename = os.path.abspath(self.filename) + To add a file without any measured data, use :meth:`touch_file`. + + You write to a named file with :meth:`write_file`, or to an already opened + file with :meth:`write_fileobj`. + + You can clear the data in memory with :meth:`erase`. Two data collections + can be combined by using :meth:`update` on one :class:`CoverageData`, + passing it the other. + + """ + + # The data file format is JSON, with these keys: + # + # * lines: a dict mapping file names to lists of line numbers + # executed:: + # + # { "file1": [17,23,45], "file2": [1,2,3], ... } + # + # * arcs: a dict mapping file names to lists of line number pairs:: + # + # { "file1": [[17,23], [17,25], [25,26]], ... } + # + # * file_tracers: a dict mapping file names to plugin names:: + # + # { "file1": "django.coverage", ... } + # + # * runs: a list of dicts of information about the coverage.py runs + # contributing to the data:: + # + # [ { "brief_sys": "CPython 2.7.10 Darwin" }, ... ] + # + # Only one of `lines` or `arcs` will be present: with branch coverage, data + # is stored as arcs. Without branch coverage, it is stored as lines. The + # line data is easily recovered from the arcs: it is all the first elements + # of the pairs that are greater than zero. + + def __init__(self, debug=None): + """Create a CoverageData. + + `debug` is a `DebugControl` object for writing debug messages. + + """ + self._debug = debug # A map from canonical Python source file name to a dictionary in # which there's an entry for each line number that has been # executed: # - # { - # 'filename1.py': { 12: None, 47: None, ... }, - # ... - # } + # { 'filename1.py': [12, 47, 1001], ... } # - self.lines = {} + self._lines = None # A map from canonical Python source file name to a dictionary with an # entry for each pair of line numbers forming an arc: # - # { - # 'filename1.py': { (12,14): None, (47,48): None, ... }, - # ... - # } + # { 'filename1.py': [(12,14), (47,48), ... ], ... } # - self.arcs = {} + self._arcs = None # A map from canonical source file name to a plugin module name: # - # { - # 'filename1.py': 'django.coverage', - # ... - # } - self.plugins = {} - - def usefile(self, use_file=True): - """Set whether or not to use a disk file for data.""" - self.use_file = use_file - - def read(self): - """Read coverage data from the coverage data file (if it exists).""" - if self.use_file: - self.lines, self.arcs, self.plugins = self._read_file(self.filename) - else: - self.lines, self.arcs, self.plugins = {}, {}, {} + # { 'filename1.py': 'django.coverage', ... } + # + self._file_tracers = {} - def write(self, suffix=None): - """Write the collected coverage data to a file. + # A list of dicts of information about the coverage.py runs. + self._runs = [] - `suffix` is a suffix to append to the base file name. This can be used - for multiple or parallel execution, so that many coverage data files - can exist simultaneously. A dot will be used to join the base name and - the suffix. + def __repr__(self): + return "<{klass} lines={lines} arcs={arcs} tracers={tracers} runs={runs}>".format( + klass=self.__class__.__name__, + lines="None" if self._lines is None else "{{{0}}}".format(len(self._lines)), + arcs="None" if self._arcs is None else "{{{0}}}".format(len(self._arcs)), + tracers="{{{0}}}".format(len(self._file_tracers)), + runs="[{0}]".format(len(self._runs)), + ) + + ## + ## Reading data + ## + + def has_arcs(self): + """Does this data have arcs? + + Arc data is only available if branch coverage was used during + collection. + + Returns a boolean. """ - if self.use_file: - filename = self.filename - if suffix: - filename += "." + suffix - self.write_file(filename) + return self._has_arcs() - def erase(self): - """Erase the data, both in this object, and from its file storage.""" - if self.use_file: - if self.filename: - file_be_gone(self.filename) - self.lines = {} - self.arcs = {} - self.plugins = {} - - def line_data(self): - """Return the map from filenames to lists of line numbers executed.""" - return dict( - (f, sorted(lmap.keys())) for f, lmap in iitems(self.lines) - ) + def lines(self, filename): + """Get the list of lines executed for a file. - def arc_data(self): - """Return the map from filenames to lists of line number pairs.""" - return dict( - (f, sorted(amap.keys())) for f, amap in iitems(self.arcs) - ) + If the file was not measured, returns None. A file might be measured, + and have no lines executed, in which case an empty list is returned. - def plugin_data(self): - return self.plugins + If the file was executed, returns a list of integers, the line numbers + executed in the file. The list is in no particular order. - def write_file(self, filename): - """Write the coverage data to `filename`.""" + """ + if self._arcs is not None: + if filename in self._arcs: + return [s for s, __ in self._arcs[filename] if s > 0] + elif self._lines is not None: + if filename in self._lines: + return self._lines[filename] + return None + + def arcs(self, filename): + """Get the list of arcs executed for a file. + + If the file was not measured, returns None. A file might be measured, + and have no arcs executed, in which case an empty list is returned. + + If the file was executed, returns a list of 2-tuples of integers. Each + pair is a starting line number and an ending line number for a + transition from one line to another. The list is in no particular + order. + + Negative numbers have special meaning. If the starting line number is + -N, it represents an entry to the code object that starts at line N. + If the ending ling number is -N, it's an exit from the code object that + starts at line N. - # Create the file data. - data = {} + """ + if self._arcs is not None: + if filename in self._arcs: + return self._arcs[filename] + return None - data['lines'] = self.line_data() - arcs = self.arc_data() - if arcs: - data['arcs'] = arcs + def file_tracer(self, filename): + """Get the plugin name of the file tracer for a file. - if self.collector: - data['collector'] = self.collector + Returns the name of the plugin that handles this file. If the file was + measured, but didn't use a plugin, then "" is returned. If the file + was not measured, then None is returned. - data['plugins'] = self.plugins + """ + # Because the vast majority of files involve no plugin, we don't store + # them explicitly in self._file_tracers. Check the measured data + # instead to see if it was a known file with no plugin. + if filename in (self._arcs or self._lines or {}): + return self._file_tracers.get(filename, "") + return None - if self.debug and self.debug.should('dataio'): - self.debug.write("Writing data to %r" % (filename,)) + def run_infos(self): + """Return the list of dicts of run information. - # Write the pickle to the file. - with open(filename, 'wb') as fdata: - pickle.dump(data, fdata, 2) + For data collected during a single run, this will be a one-element + list. If data has been combined, there will be one element for each + original data file. - def read_file(self, filename): - """Read the coverage data from `filename`.""" - self.lines, self.arcs, self.plugins = self._read_file(filename) + """ + return self._runs + + def measured_files(self): + """A list of all files that had been measured.""" + return list(self._arcs or self._lines or {}) - def raw_data(self, filename): - """Return the raw pickled data from `filename`.""" - if self.debug and self.debug.should('dataio'): - self.debug.write("Reading data from %r" % (filename,)) - with open(filename, 'rb') as fdata: - data = pickle.load(fdata) - return data + def line_counts(self, fullpath=False): + """Return a dict summarizing the line coverage data. - def _read_file(self, filename): - """Return the stored coverage data from the given file. + Keys are based on the file names, and values are the number of executed + lines. If `fullpath` is true, then the keys are the full pathnames of + the files, otherwise they are the basenames of the files. - Returns three values, suitable for assigning to `self.lines`, - `self.arcs`, and `self.plugins`. + Returns a dict mapping file names to counts of lines. """ - lines = {} - arcs = {} - plugins = {} - try: - data = self.raw_data(filename) - if isinstance(data, dict): - # Unpack the 'lines' item. - lines = dict([ - (f, dict.fromkeys(linenos, None)) - for f, linenos in iitems(data.get('lines', {})) - ]) - # Unpack the 'arcs' item. - arcs = dict([ - (f, dict.fromkeys(arcpairs, None)) - for f, arcpairs in iitems(data.get('arcs', {})) - ]) - plugins = data.get('plugins', {}) - except Exception: - pass - return lines, arcs, plugins - - def combine_parallel_data(self, aliases=None, data_dirs=None): - """Combine a number of data files together. + summ = {} + if fullpath: + filename_fn = lambda f: f + else: + filename_fn = os.path.basename + for filename in self.measured_files(): + summ[filename_fn(filename)] = len(self.lines(filename)) + return summ - Treat `self.filename` as a file prefix, and combine the data from all - of the data files starting with that prefix plus a dot. + def __nonzero__(self): + return bool(self._lines or self._arcs) - If `aliases` is provided, it's a `PathAliases` object that is used to - re-map paths to match the local machine's. + __bool__ = __nonzero__ + + def read_fileobj(self, file_obj): + """Read the coverage data from the given file object. - If `data_dirs` is provided, then it combines the data files from each - directory into a single file. + Should only be used on an empty CoverageData object. """ - aliases = aliases or PathAliases() - data_dir, local = os.path.split(self.filename) - localdot = local + '.*' + data = self._read_raw_data(file_obj) - data_dirs = data_dirs or [data_dir] - files_to_combine = [] - for d in data_dirs: - pattern = os.path.join(os.path.abspath(d), localdot) - files_to_combine.extend(glob.glob(pattern)) + self._lines = self._arcs = None - for f in files_to_combine: - new_lines, new_arcs, new_plugins = self._read_file(f) - for filename, file_data in iitems(new_lines): - filename = aliases.map(filename) - self.lines.setdefault(filename, {}).update(file_data) - for filename, file_data in iitems(new_arcs): - filename = aliases.map(filename) - self.arcs.setdefault(filename, {}).update(file_data) - self.plugins.update(new_plugins) - os.remove(f) + if 'lines' in data: + self._lines = data['lines'] + if 'arcs' in data: + self._arcs = dict( + (fname, [tuple(pair) for pair in arcs]) + for fname, arcs in iitems(data['arcs']) + ) + self._file_tracers = data.get('file_tracers', {}) + self._runs = data.get('runs', []) - def add_line_data(self, line_data): - """Add executed line data. + self._validate() - `line_data` is { filename: { lineno: None, ... }, ...} + def read_file(self, filename): + """Read the coverage data from `filename` into this object.""" + if self._debug and self._debug.should('dataio'): + self._debug.write("Reading data from %r" % (filename,)) + try: + with self._open_for_reading(filename) as f: + self.read_fileobj(f) + except Exception as exc: + raise CoverageException( + "Couldn't read data from '%s': %s: %s" % ( + filename, exc.__class__.__name__, exc, + ) + ) + + _GO_AWAY = "!coverage.py: This is a private format, don't read it directly!" + + @classmethod + def _open_for_reading(cls, filename): + """Open a file appropriately for reading data.""" + return open(filename, "r") + + @classmethod + def _read_raw_data(cls, file_obj): + """Read the raw data from a file object.""" + go_away = file_obj.read(len(cls._GO_AWAY)) + if go_away != cls._GO_AWAY: + raise CoverageException("Doesn't seem to be a coverage.py data file") + return json.load(file_obj) + + @classmethod + def _read_raw_data_file(cls, filename): + """Read the raw data from a file, for debugging.""" + with cls._open_for_reading(filename) as f: + return cls._read_raw_data(f) + + ## + ## Writing data + ## + + def add_lines(self, line_data): + """Add measured line data. + + `line_data` is a dictionary mapping file names to dictionaries:: + + { filename: { lineno: None, ... }, ...} """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding lines: %d files, %d lines total" % ( + len(line_data), sum(len(lines) for lines in line_data.values()) + )) + if self._has_arcs(): + raise CoverageException("Can't add lines to existing arc data") + + if self._lines is None: + self._lines = {} for filename, linenos in iitems(line_data): - self.lines.setdefault(filename, {}).update(linenos) + if filename in self._lines: + new_linenos = set(self._lines[filename]) + new_linenos.update(linenos) + linenos = new_linenos + self._lines[filename] = list(linenos) - def add_arc_data(self, arc_data): + self._validate() + + def add_arcs(self, arc_data): """Add measured arc data. - `arc_data` is { filename: { (l1,l2): None, ... }, ...} + `arc_data` is a dictionary mapping file names to dictionaries:: + + { filename: { (l1,l2): None, ... }, ...} """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding arcs: %d files, %d arcs total" % ( + len(arc_data), sum(len(arcs) for arcs in arc_data.values()) + )) + if self._has_lines(): + raise CoverageException("Can't add arcs to existing line data") + + if self._arcs is None: + self._arcs = {} for filename, arcs in iitems(arc_data): - self.arcs.setdefault(filename, {}).update(arcs) + if filename in self._arcs: + new_arcs = set(self._arcs[filename]) + new_arcs.update(arcs) + arcs = new_arcs + self._arcs[filename] = list(arcs) + + self._validate() - def add_plugin_data(self, plugin_data): - self.plugins.update(plugin_data) + def add_file_tracers(self, file_tracers): + """Add per-file plugin information. + + `file_tracers` is { filename: plugin_name, ... } + + """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding file tracers: %d files" % (len(file_tracers),)) + + existing_files = self._arcs or self._lines or {} + for filename, plugin_name in iitems(file_tracers): + if filename not in existing_files: + raise CoverageException( + "Can't add file tracer data for unmeasured file '%s'" % (filename,) + ) + existing_plugin = self._file_tracers.get(filename) + if existing_plugin is not None and plugin_name != existing_plugin: + raise CoverageException( + "Conflicting file tracer name for '%s': %r vs %r" % ( + filename, existing_plugin, plugin_name, + ) + ) + self._file_tracers[filename] = plugin_name + + self._validate() + + def add_run_info(self, **kwargs): + """Add information about the run. + + Keywords are arbitrary, and are stored in the run dictionary. Values + must be JSON serializable. You may use this function more than once, + but repeated keywords overwrite each other. + + """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding run info: %r" % (kwargs,)) + if not self._runs: + self._runs = [{}] + self._runs[0].update(kwargs) + self._validate() def touch_file(self, filename): """Ensure that `filename` appears in the data, empty if needed.""" - self.lines.setdefault(filename, {}) + if self._debug and self._debug.should('dataop'): + self._debug.write("Touching %r" % (filename,)) + if not self._has_arcs() and not self._has_lines(): + raise CoverageException("Can't touch files in an empty CoverageData") - def measured_files(self): - """A list of all files that had been measured.""" - return list(self.lines.keys()) + if self._has_arcs(): + where = self._arcs + else: + where = self._lines + where.setdefault(filename, []) - def executed_lines(self, filename): - """A map containing all the line numbers executed in `filename`. + self._validate() - If `filename` hasn't been collected at all (because it wasn't executed) - then return an empty map. + def write_fileobj(self, file_obj): + """Write the coverage data to `file_obj`.""" + + # Create the file data. + file_data = {} + + if self._has_arcs(): + file_data['arcs'] = self._arcs + + if self._has_lines(): + file_data['lines'] = self._lines + + if self._file_tracers: + file_data['file_tracers'] = self._file_tracers + + if self._runs: + file_data['runs'] = self._runs + + # Write the data to the file. + file_obj.write(self._GO_AWAY) + json.dump(file_data, file_obj) + + def write_file(self, filename): + """Write the coverage data to `filename`.""" + if self._debug and self._debug.should('dataio'): + self._debug.write("Writing data to %r" % (filename,)) + with open(filename, 'w') as fdata: + self.write_fileobj(fdata) + + def erase(self): + """Erase the data in this object.""" + self._lines = None + self._arcs = None + self._file_tracers = {} + self._runs = [] + self._validate() + + def update(self, other_data, aliases=None): + """Update this data with data from another `CoverageData`. + + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. """ - return self.lines.get(filename) or {} + if self._has_lines() and other_data._has_arcs(): + raise CoverageException("Can't combine arc data with line data") + if self._has_arcs() and other_data._has_lines(): + raise CoverageException("Can't combine line data with arc data") - def executed_arcs(self, filename): - """A map containing all the arcs executed in `filename`.""" - return self.arcs.get(filename) or {} + aliases = aliases or PathAliases() - def add_to_hash(self, filename, hasher): - """Contribute `filename`'s data to the Md5Hash `hasher`.""" - hasher.update(self.executed_lines(filename)) - hasher.update(self.executed_arcs(filename)) + # _file_tracers: only have a string, so they have to agree. + # Have to do these first, so that our examination of self._arcs and + # self._lines won't be confused by data updated from other_data. + for filename in other_data.measured_files(): + other_plugin = other_data.file_tracer(filename) + filename = aliases.map(filename) + this_plugin = self.file_tracer(filename) + if this_plugin is None: + if other_plugin: + self._file_tracers[filename] = other_plugin + elif this_plugin != other_plugin: + raise CoverageException( + "Conflicting file tracer name for '%s': %r vs %r" % ( + filename, this_plugin, other_plugin, + ) + ) + + # _runs: add the new runs to these runs. + self._runs.extend(other_data._runs) + + # _lines: merge dicts. + if other_data._has_lines(): + if self._lines is None: + self._lines = {} + for filename, file_lines in iitems(other_data._lines): + filename = aliases.map(filename) + if filename in self._lines: + lines = set(self._lines[filename]) + lines.update(file_lines) + file_lines = list(lines) + self._lines[filename] = file_lines + + # _arcs: merge dicts. + if other_data._has_arcs(): + if self._arcs is None: + self._arcs = {} + for filename, file_arcs in iitems(other_data._arcs): + filename = aliases.map(filename) + if filename in self._arcs: + arcs = set(self._arcs[filename]) + arcs.update(file_arcs) + file_arcs = list(arcs) + self._arcs[filename] = file_arcs + + self._validate() + + ## + ## Miscellaneous + ## + + def _validate(self): + """If we are in paranoid mode, validate that everything is right.""" + if env.TESTING: + self._validate_invariants() + + def _validate_invariants(self): + """Validate internal invariants.""" + # Only one of _lines or _arcs should exist. + assert not(self._has_lines() and self._has_arcs()), ( + "Shouldn't have both _lines and _arcs" + ) + + # _lines should be a dict of lists of ints. + if self._has_lines(): + for fname, lines in iitems(self._lines): + assert isinstance(fname, string_class), "Key in _lines shouldn't be %r" % (fname,) + assert all(isinstance(x, int) for x in lines), ( + "_lines[%r] shouldn't be %r" % (fname, lines) + ) + + # _arcs should be a dict of lists of pairs of ints. + if self._has_arcs(): + for fname, arcs in iitems(self._arcs): + assert isinstance(fname, string_class), "Key in _arcs shouldn't be %r" % (fname,) + assert all(isinstance(x, int) and isinstance(y, int) for x, y in arcs), ( + "_arcs[%r] shouldn't be %r" % (fname, arcs) + ) + + # _file_tracers should have only non-empty strings as values. + for fname, plugin in iitems(self._file_tracers): + assert isinstance(fname, string_class), ( + "Key in _file_tracers shouldn't be %r" % (fname,) + ) + assert plugin and isinstance(plugin, string_class), ( + "_file_tracers[%r] shoudn't be %r" % (fname, plugin) + ) - def summary(self, fullpath=False): - """Return a dict summarizing the coverage data. + # _runs should be a list of dicts. + for val in self._runs: + assert isinstance(val, dict) + for key in val: + assert isinstance(key, string_class), "Key in _runs shouldn't be %r" % (key,) - Keys are based on the filenames, and values are the number of executed - lines. If `fullpath` is true, then the keys are the full pathnames of - the files, otherwise they are the basenames of the files. + def add_to_hash(self, filename, hasher): + """Contribute `filename`'s data to the `hasher`. + + `hasher` is a `coverage.misc.Hasher` instance to be updated with + the file's data. It should only get the results data, not the run + data. """ - summ = {} - if fullpath: - filename_fn = lambda f: f + if self._has_arcs(): + hasher.update(sorted(self.arcs(filename) or [])) else: - filename_fn = os.path.basename - for filename, lines in iitems(self.lines): - summ[filename_fn(filename)] = len(lines) - return summ + hasher.update(sorted(self.lines(filename) or [])) + hasher.update(self.file_tracer(filename)) - def has_arcs(self): - """Does this data have arcs?""" - return bool(self.arcs) + ## + ## Internal + ## + + def _has_lines(self): + """Do we have data in self._lines?""" + return self._lines is not None + + def _has_arcs(self): + """Do we have data in self._arcs?""" + return self._arcs is not None + + +class CoverageDataFiles(object): + """Manage the use of coverage data files.""" + + def __init__(self, basename=None, warn=None): + """Create a CoverageDataFiles to manage data files. + + `warn` is the warning function to use. + + `basename` is the name of the file to use for storing data. + + """ + self.warn = warn + # Construct the file name that will be used for data storage. + self.filename = os.path.abspath(basename or ".coverage") + + def erase(self, parallel=False): + """Erase the data from the file storage. + + If `parallel` is true, then also deletes data files created from the + basename by parallel-mode. + + """ + file_be_gone(self.filename) + if parallel: + data_dir, local = os.path.split(self.filename) + localdot = local + '.*' + pattern = os.path.join(os.path.abspath(data_dir), localdot) + for filename in glob.glob(pattern): + file_be_gone(filename) + + def read(self, data): + """Read the coverage data.""" + if os.path.exists(self.filename): + data.read_file(self.filename) + + def write(self, data, suffix=None): + """Write the collected coverage data to a file. + + `suffix` is a suffix to append to the base file name. This can be used + for multiple or parallel execution, so that many coverage data files + can exist simultaneously. A dot will be used to join the base name and + the suffix. + + """ + filename = self.filename + if suffix is True: + # If data_suffix was a simple true value, then make a suffix with + # plenty of distinguishing information. We do this here in + # `save()` at the last minute so that the pid will be correct even + # if the process forks. + extra = "" + if _TEST_NAME_FILE: # pragma: debugging + with open(_TEST_NAME_FILE) as f: + test_name = f.read() + extra = "." + test_name + suffix = "%s%s.%s.%06d" % ( + socket.gethostname(), extra, os.getpid(), + random.randint(0, 999999) + ) + + if suffix: + filename += "." + suffix + data.write_file(filename) + + def combine_parallel_data(self, data, aliases=None, data_paths=None): + """Combine a number of data files together. + + Treat `self.filename` as a file prefix, and combine the data from all + of the data files starting with that prefix plus a dot. + + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. + + If `data_paths` is provided, it is a list of directories or files to + combine. Directories are searched for files that start with + `self.filename` plus dot as a prefix, and those files are combined. + + If `data_paths` is not provided, then the directory portion of + `self.filename` is used as the directory to search for data files. + + Every data file found and combined is then deleted from disk. If a file + cannot be read, a warning will be issued, and the file will not be + deleted. + + """ + # Because of the os.path.abspath in the constructor, data_dir will + # never be an empty string. + data_dir, local = os.path.split(self.filename) + localdot = local + '.*' + + data_paths = data_paths or [data_dir] + files_to_combine = [] + for p in data_paths: + if os.path.isfile(p): + files_to_combine.append(os.path.abspath(p)) + elif os.path.isdir(p): + pattern = os.path.join(os.path.abspath(p), localdot) + files_to_combine.extend(glob.glob(pattern)) + else: + raise CoverageException("Couldn't combine from non-existent path '%s'" % (p,)) + + for f in files_to_combine: + new_data = CoverageData() + try: + new_data.read_file(f) + except CoverageException as exc: + if self.warn: + # The CoverageException has the file name in it, so just + # use the message as the warning. + self.warn(str(exc)) + else: + data.update(new_data, aliases=aliases) + file_be_gone(f) + + +def canonicalize_json_data(data): + """Canonicalize our JSON data so it can be compared.""" + for fname, lines in iitems(data.get('lines', {})): + data['lines'][fname] = sorted(lines) + for fname, arcs in iitems(data.get('arcs', {})): + data['arcs'][fname] = sorted(arcs) + + +def pretty_data(data): + """Format data as JSON, but as nicely as possible. + + Returns a string. + + """ + # Start with a basic JSON dump. + out = json.dumps(data, indent=4, sort_keys=True) + # But pairs of numbers shouldn't be split across lines... + out = re.sub(r"\[\s+(-?\d+),\s+(-?\d+)\s+]", r"[\1, \2]", out) + # Trailing spaces mess with tests, get rid of them. + out = re.sub(r"(?m)\s+$", "", out) + return out + + +def debug_main(args): + """Dump the raw data from data files. + + Run this as:: + + $ python -m coverage.data [FILE] + + """ + parser = optparse.OptionParser() + parser.add_option( + "-c", "--canonical", action="store_true", + help="Sort data into a canonical order", + ) + options, args = parser.parse_args(args) + + for filename in (args or [".coverage"]): + print("--- {0} ------------------------------".format(filename)) + data = CoverageData._read_raw_data_file(filename) + if options.canonical: + canonicalize_json_data(data) + print(pretty_data(data)) if __name__ == '__main__': - # Ad-hoc: show the raw data in a data file. - import pprint, sys - covdata = CoverageData() - if sys.argv[1:]: - fname = sys.argv[1] - else: - fname = covdata.filename - pprint.pprint(covdata.raw_data(fname)) + import sys + debug_main(sys.argv[1:]) diff --git a/coverage/debug.py b/coverage/debug.py index 5b41bc40..4076b9b2 100644 --- a/coverage/debug.py +++ b/coverage/debug.py @@ -1,6 +1,15 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Control of and utilities for debugging.""" +import inspect import os +import sys + +from coverage.misc import isolate_module + +os = isolate_module(os) # When debugging, it can be helpful to force some options, especially when @@ -8,6 +17,9 @@ import os # This is a list of forced debugging options. FORCED_DEBUG = [] +# A hack for debugging testing in sub-processes. +_TEST_NAME_FILE = "" # "/tmp/covtest.txt" + class DebugControl(object): """Control and output for debugging.""" @@ -18,9 +30,7 @@ class DebugControl(object): self.output = output def __repr__(self): - return "<DebugControl options=%r output=%r>" % ( - self.options, self.output - ) + return "<DebugControl options=%r output=%r>" % (self.options, self.output) def should(self, option): """Decide whether to output debug information in category `option`.""" @@ -31,6 +41,8 @@ class DebugControl(object): if self.should('pid'): msg = "pid %5d: %s" % (os.getpid(), msg) self.output.write(msg+"\n") + if self.should('callers'): + dump_stack_frames(self.output) self.output.flush() def write_formatted_info(self, header, info): @@ -66,3 +78,27 @@ def info_formatter(info): prefix = "" else: yield "%*s: %s" % (label_len, label, data) + + +def short_stack(): # pragma: debugging + """Return a string summarizing the call stack. + + The string is multi-line, with one line per stack frame. Each line shows + the function name, the file name, and the line number: + + ... + start_import_stop : /Users/ned/coverage/trunk/tests/coveragetest.py @95 + import_local_file : /Users/ned/coverage/trunk/tests/coveragetest.py @81 + import_local_file : /Users/ned/coverage/trunk/coverage/backward.py @159 + ... + + """ + stack = inspect.stack()[:0:-1] + return "\n".join("%30s : %s @%d" % (t[3], t[1], t[2]) for t in stack) + + +def dump_stack_frames(out=None): # pragma: debugging + """Print a summary of the stack to stdout, or some place else.""" + out = out or sys.stdout + out.write(short_stack()) + out.write("\n") diff --git a/coverage/env.py b/coverage/env.py index 85ffa5ff..4cd02c04 100644 --- a/coverage/env.py +++ b/coverage/env.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Determine facts about the environment.""" import os @@ -11,11 +14,19 @@ LINUX = sys.platform == "linux2" PYPY = '__pypy__' in sys.builtin_module_names # Python versions. -PY2 = sys.version_info < (3, 0) -PY3 = sys.version_info >= (3, 0) +PYVERSION = sys.version_info +PY2 = PYVERSION < (3, 0) +PY3 = PYVERSION >= (3, 0) # Coverage.py specifics. + # Are we using the C-implemented trace function? C_TRACER = os.getenv('COVERAGE_TEST_TRACER', 'c') == 'c' + # Are we coverage-measuring ourselves? METACOV = os.getenv('COVERAGE_COVERAGE', '') != '' + +# Are we running our test suite? +# Even when running tests, you can use COVERAGE_TESTING=0 to disable the +# test-specific behavior like contracts. +TESTING = os.getenv('COVERAGE_TESTING', '') == 'True' diff --git a/coverage/execfile.py b/coverage/execfile.py index 2d856897..3e20a527 100644 --- a/coverage/execfile.py +++ b/coverage/execfile.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Execute files of Python code.""" import marshal @@ -7,9 +10,12 @@ import types from coverage.backward import BUILTINS from coverage.backward import PYC_MAGIC_NUMBER, imp, importlib_util_find_spec -from coverage.misc import ExceptionDuringRun, NoCode, NoSource +from coverage.misc import ExceptionDuringRun, NoCode, NoSource, isolate_module +from coverage.phystokens import compile_unicode from coverage.python import get_python_source +os = isolate_module(os) + class DummyLoader(object): """A shim for the pep302 __loader__, emulating pkgutil.ImpLoader. @@ -35,7 +41,7 @@ if importlib_util_find_spec: raise NoSource("No module named %r" % (modulename,)) pathname = spec.origin packagename = spec.name - if pathname.endswith("__init__.py"): + if pathname.endswith("__init__.py") and not modulename.endswith("__init__"): mod_main = modulename + ".__main__" spec = importlib_util_find_spec(mod_main) if not spec: @@ -104,10 +110,10 @@ def run_python_module(modulename, args): pathname = os.path.abspath(pathname) args[0] = pathname - run_python_file(pathname, args, package=packagename, modulename=modulename) + run_python_file(pathname, args, package=packagename, modulename=modulename, path0="") -def run_python_file(filename, args, package=None, modulename=None): +def run_python_file(filename, args, package=None, modulename=None, path0=None): """Run a Python file as if it were the main program on the command line. `filename` is the path to the file to execute, it need not be a .py file. @@ -117,6 +123,9 @@ def run_python_file(filename, args, package=None, modulename=None): `modulename` is the name of the module the file was run as. + `path0` is the value to put into sys.path[0]. If it's None, then this + function will decide on a value. + """ if modulename is None and sys.version_info >= (3, 3): modulename = '__main__' @@ -137,6 +146,25 @@ def run_python_file(filename, args, package=None, modulename=None): old_argv = sys.argv sys.argv = args + if os.path.isdir(filename): + # Running a directory means running the __main__.py file in that + # directory. + my_path0 = filename + + for ext in [".py", ".pyc", ".pyo"]: + try_filename = os.path.join(filename, "__main__" + ext) + if os.path.exists(try_filename): + filename = try_filename + break + else: + raise NoSource("Can't find '__main__' module in '%s'" % filename) + else: + my_path0 = os.path.abspath(os.path.dirname(filename)) + + # Set sys.path correctly. + old_path0 = sys.path[0] + sys.path[0] = path0 if path0 is not None else my_path0 + try: # Make a code object somehow. if filename.endswith((".pyc", ".pyo")): @@ -167,11 +195,10 @@ def run_python_file(filename, args, package=None, modulename=None): raise ExceptionDuringRun(typ, err, tb.tb_next) finally: - # Restore the old __main__ + # Restore the old __main__, argv, and path. sys.modules['__main__'] = old_main_mod - - # Restore the old argv and path sys.argv = old_argv + sys.path[0] = old_path0 def make_code_from_py(filename): @@ -182,7 +209,7 @@ def make_code_from_py(filename): except (IOError, NoSource): raise NoSource("No file to run: '%s'" % filename) - code = compile(source, filename, "exec") + code = compile_unicode(source, filename, "exec") return code diff --git a/coverage/files.py b/coverage/files.py index f7fc9693..44997d12 100644 --- a/coverage/files.py +++ b/coverage/files.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """File wrangling.""" import fnmatch @@ -9,100 +12,152 @@ import re import sys from coverage import env -from coverage.misc import CoverageException, join_regex +from coverage.backward import unicode_class +from coverage.misc import contract, CoverageException, join_regex, isolate_module -class FileLocator(object): - """Understand how filenames work.""" +os = isolate_module(os) - def __init__(self): - # The absolute path to our current directory. - self.relative_dir = os.path.normcase(abs_file(os.curdir) + os.sep) - # Cache of results of calling the canonical_filename() method, to - # avoid duplicating work. - self.canonical_filename_cache = {} +def set_relative_directory(): + """Set the directory that `relative_filename` will be relative to.""" + global RELATIVE_DIR, CANONICAL_FILENAME_CACHE - def relative_filename(self, filename): - """Return the relative form of `filename`. + # The absolute path to our current directory. + RELATIVE_DIR = os.path.normcase(abs_file(os.curdir) + os.sep) - The filename will be relative to the current directory when the - `FileLocator` was constructed. + # Cache of results of calling the canonical_filename() method, to + # avoid duplicating work. + CANONICAL_FILENAME_CACHE = {} - """ - fnorm = os.path.normcase(filename) - if fnorm.startswith(self.relative_dir): - filename = filename[len(self.relative_dir):] - return filename - def canonical_filename(self, filename): - """Return a canonical filename for `filename`. +def relative_directory(): + """Return the directory that `relative_filename` is relative to.""" + return RELATIVE_DIR - An absolute path with no redundant components and normalized case. - """ - if filename not in self.canonical_filename_cache: - if not os.path.isabs(filename): - for path in [os.curdir] + sys.path: - if path is None: - continue - f = os.path.join(path, filename) - if os.path.exists(f): - filename = f - break - cf = abs_file(filename) - self.canonical_filename_cache[filename] = cf - return self.canonical_filename_cache[filename] +@contract(returns='unicode') +def relative_filename(filename): + """Return the relative form of `filename`. + + The file name will be relative to the current directory when the + `set_relative_directory` was called. + + """ + fnorm = os.path.normcase(filename) + if fnorm.startswith(RELATIVE_DIR): + filename = filename[len(RELATIVE_DIR):] + return unicode_filename(filename) + + +@contract(returns='unicode') +def canonical_filename(filename): + """Return a canonical file name for `filename`. + + An absolute path with no redundant components and normalized case. + + """ + if filename not in CANONICAL_FILENAME_CACHE: + if not os.path.isabs(filename): + for path in [os.curdir] + sys.path: + if path is None: + continue + f = os.path.join(path, filename) + if os.path.exists(f): + filename = f + break + cf = abs_file(filename) + CANONICAL_FILENAME_CACHE[filename] = cf + return CANONICAL_FILENAME_CACHE[filename] + + +def flat_rootname(filename): + """A base for a flat file name to correspond to this file. + + Useful for writing files about the code where you want all the files in + the same directory, but need to differentiate same-named files from + different directories. + + For example, the file a/b/c.py will return 'a_b_c_py' + + """ + name = ntpath.splitdrive(filename)[1] + return re.sub(r"[\\/.:]", "_", name) if env.WINDOWS: + _ACTUAL_PATH_CACHE = {} + _ACTUAL_PATH_LIST_CACHE = {} + def actual_path(path): """Get the actual path of `path`, including the correct case.""" - if path in actual_path.cache: - return actual_path.cache[path] + if env.PY2 and isinstance(path, unicode_class): + path = path.encode(sys.getfilesystemencoding()) + if path in _ACTUAL_PATH_CACHE: + return _ACTUAL_PATH_CACHE[path] head, tail = os.path.split(path) if not tail: - actpath = head + # This means head is the drive spec: normalize it. + actpath = head.upper() elif not head: actpath = tail else: head = actual_path(head) - if head in actual_path.list_cache: - files = actual_path.list_cache[head] + if head in _ACTUAL_PATH_LIST_CACHE: + files = _ACTUAL_PATH_LIST_CACHE[head] else: try: files = os.listdir(head) except OSError: files = [] - actual_path.list_cache[head] = files + _ACTUAL_PATH_LIST_CACHE[head] = files normtail = os.path.normcase(tail) for f in files: if os.path.normcase(f) == normtail: tail = f break actpath = os.path.join(head, tail) - actual_path.cache[path] = actpath + _ACTUAL_PATH_CACHE[path] = actpath return actpath - actual_path.cache = {} - actual_path.list_cache = {} - else: def actual_path(filename): """The actual path for non-Windows platforms.""" return filename +if env.PY2: + @contract(returns='unicode') + def unicode_filename(filename): + """Return a Unicode version of `filename`.""" + if isinstance(filename, str): + encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + filename = filename.decode(encoding, "replace") + return filename +else: + @contract(filename='unicode', returns='unicode') + def unicode_filename(filename): + """Return a Unicode version of `filename`.""" + return filename + + +@contract(returns='unicode') def abs_file(filename): """Return the absolute normalized form of `filename`.""" path = os.path.expandvars(os.path.expanduser(filename)) path = os.path.abspath(os.path.realpath(path)) path = actual_path(path) + path = unicode_filename(path) return path +RELATIVE_DIR = None +CANONICAL_FILENAME_CACHE = None +set_relative_directory() + + def isabs_anywhere(filename): """Is `filename` an absolute path on any OS?""" return ntpath.isabs(filename) or posixpath.isabs(filename) @@ -181,7 +236,7 @@ class ModuleMatcher(object): class FnmatchMatcher(object): - """A matcher for files by filename pattern.""" + """A matcher for files by file name pattern.""" def __init__(self, pats): self.pats = pats[:] # fnmatch is platform-specific. On Windows, it does the Windows thing @@ -204,7 +259,7 @@ class FnmatchMatcher(object): return self.pats def match(self, fpath): - """Does `fpath` match one of our filename patterns?""" + """Does `fpath` match one of our file name patterns?""" return self.re.match(fpath) is not None @@ -228,12 +283,9 @@ class PathAliases(object): A `PathAliases` object tracks a list of pattern/result pairs, and can map a path through those aliases to produce a unified path. - `locator` is a FileLocator that is used to canonicalize the results. - """ - def __init__(self, locator=None): + def __init__(self): self.aliases = [] - self.locator = locator def add(self, pattern, result): """Add the `pattern`/`result` pair to the list of aliases. @@ -286,6 +338,10 @@ class PathAliases(object): The separator style in the result is made to match that of the result in the alias. + Returns the mapped path. If a mapping has happened, this is a + canonical path. If no mapping has happened, it is the original value + of `path` unchanged. + """ for regex, result, pattern_sep, result_sep in self.aliases: m = regex.match(path) @@ -293,8 +349,7 @@ class PathAliases(object): new = path.replace(m.group(0), result) if pattern_sep != result_sep: new = new.replace(pattern_sep, result_sep) - if self.locator: - new = self.locator.canonical_filename(new) + new = canonical_filename(new) return new return path diff --git a/coverage/fullcoverage/encodings.py b/coverage/fullcoverage/encodings.py index 6a258d67..699f3863 100644 --- a/coverage/fullcoverage/encodings.py +++ b/coverage/fullcoverage/encodings.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Imposter encodings module that installs a coverage-style tracer. This is NOT the encodings module; it is an imposter that sets up tracing @@ -6,10 +9,10 @@ instrumentation and then replaces itself with the real encodings module. If the directory that holds this file is placed first in the PYTHONPATH when using "coverage" to run Python's tests, then this file will become the very first module imported by the internals of Python 3. It installs a -coverage-compatible trace function that can watch Standard Library modules +coverage.py-compatible trace function that can watch Standard Library modules execute from the very earliest stages of Python's own boot process. This fixes -a problem with coverage - that it starts too late to trace the coverage of many -of the most fundamental modules in the Standard Library. +a problem with coverage.py - that it starts too late to trace the coverage of +many of the most fundamental modules in the Standard Library. """ diff --git a/coverage/html.py b/coverage/html.py index 0b2cc25c..8dca6323 100644 --- a/coverage/html.py +++ b/coverage/html.py @@ -1,6 +1,7 @@ -"""HTML reporting for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt -from __future__ import unicode_literals +"""HTML reporting for coverage.py.""" import datetime import json @@ -11,11 +12,14 @@ import shutil import coverage from coverage import env from coverage.backward import iitems -from coverage.misc import CoverageException, Hasher +from coverage.files import flat_rootname +from coverage.misc import CoverageException, Hasher, isolate_module from coverage.report import Reporter from coverage.results import Numbers from coverage.templite import Templite +os = isolate_module(os) + # Static files are looked for in a list of places. STATIC_PATH = [ @@ -26,6 +30,7 @@ STATIC_PATH = [ os.path.join(os.path.dirname(__file__), "htmlfiles"), ] + def data_filename(fname, pkgdir=""): """Return the path to a data file of ours. @@ -36,15 +41,22 @@ def data_filename(fname, pkgdir=""): is provided, at that sub-directory. """ + tried = [] for static_dir in STATIC_PATH: static_filename = os.path.join(static_dir, fname) if os.path.exists(static_filename): return static_filename + else: + tried.append(static_filename) if pkgdir: static_filename = os.path.join(static_dir, pkgdir, fname) if os.path.exists(static_filename): return static_filename - raise CoverageException("Couldn't find static file %r" % fname) + else: + tried.append(static_filename) + raise CoverageException( + "Couldn't find static file %r from %r, tried: %r" % (fname, os.getcwd(), tried) + ) def data(fname): @@ -56,18 +68,19 @@ def data(fname): class HtmlReporter(Reporter): """HTML reporting.""" - # These files will be copied from the htmlfiles dir to the output dir. + # These files will be copied from the htmlfiles directory to the output + # directory. STATIC_FILES = [ - ("style.css", ""), - ("jquery.min.js", "jquery"), - ("jquery.debounce.min.js", "jquery-debounce"), - ("jquery.hotkeys.js", "jquery-hotkeys"), - ("jquery.isonscreen.js", "jquery-isonscreen"), - ("jquery.tablesorter.min.js", "jquery-tablesorter"), - ("coverage_html.js", ""), - ("keybd_closed.png", ""), - ("keybd_open.png", ""), - ] + ("style.css", ""), + ("jquery.min.js", "jquery"), + ("jquery.debounce.min.js", "jquery-debounce"), + ("jquery.hotkeys.js", "jquery-hotkeys"), + ("jquery.isonscreen.js", "jquery-isonscreen"), + ("jquery.tablesorter.min.js", "jquery-tablesorter"), + ("coverage_html.js", ""), + ("keybd_closed.png", ""), + ("keybd_open.png", ""), + ] def __init__(self, cov, config): super(HtmlReporter, self).__init__(cov, config) @@ -81,15 +94,15 @@ class HtmlReporter(Reporter): 'title': title, '__url__': coverage.__url__, '__version__': coverage.__version__, - } + } self.source_tmpl = Templite( data("pyfile.html"), self.template_globals - ) + ) self.coverage = cov self.files = [] - self.arcs = self.coverage.data.has_arcs() + self.has_arcs = self.coverage.data.has_arcs() self.status = HtmlStatus() self.extra_css = None self.totals = Numbers() @@ -98,7 +111,7 @@ class HtmlReporter(Reporter): def report(self, morfs): """Generate an HTML report for `morfs`. - `morfs` is a list of modules or filenames. + `morfs` is a list of modules or file names. """ assert self.config.html_dir, "must give a directory for html reporting" @@ -128,8 +141,7 @@ class HtmlReporter(Reporter): self.index_file() self.make_local_static_report_files() - - return self.totals.pc_covered + return self.totals.n_statements and self.totals.pc_covered def make_local_static_report_files(self): """Make local instances of static files for HTML report.""" @@ -138,14 +150,14 @@ class HtmlReporter(Reporter): shutil.copyfile( data_filename(static, pkgdir), os.path.join(self.directory, static) - ) + ) # The user may have extra CSS they want copied. if self.extra_css: shutil.copyfile( self.config.extra_css, os.path.join(self.directory, self.extra_css) - ) + ) def write_html(self, fname, html): """Write `html` to `fname`, properly encoded.""" @@ -164,20 +176,20 @@ class HtmlReporter(Reporter): source = fr.source() # Find out if the file on disk is already correct. - flat_rootname = fr.flat_rootname() + rootname = flat_rootname(fr.relative_filename()) this_hash = self.file_hash(source.encode('utf-8'), fr) - that_hash = self.status.file_hash(flat_rootname) + that_hash = self.status.file_hash(rootname) if this_hash == that_hash: # Nothing has changed to require the file to be reported again. - self.files.append(self.status.index_info(flat_rootname)) + self.files.append(self.status.index_info(rootname)) return - self.status.set_file_hash(flat_rootname, this_hash) + self.status.set_file_hash(rootname, this_hash) # Get the numbers for this file. nums = analysis.numbers - if self.arcs: + if self.has_arcs: missing_branch_arcs = analysis.missing_branch_arcs() # These classes determine which lines are highlighted by default. @@ -199,23 +211,34 @@ class HtmlReporter(Reporter): line_class.append(c_exc) elif lineno in analysis.missing: line_class.append(c_mis) - elif self.arcs and lineno in missing_branch_arcs: + elif self.has_arcs and lineno in missing_branch_arcs: line_class.append(c_par) - annlines = [] + shorts = [] + longs = [] for b in missing_branch_arcs[lineno]: if b < 0: - annlines.append("exit") + shorts.append("exit") + longs.append("the function exit") else: - annlines.append(str(b)) - annotate_html = " ".join(annlines) - if len(annlines) > 1: - annotate_title = "no jumps to these line numbers" - elif len(annlines) == 1: - annotate_title = "no jump to this line number" + shorts.append(b) + longs.append("line %d" % b) + # 202F is NARROW NO-BREAK SPACE. + # 219B is RIGHTWARDS ARROW WITH STROKE. + short_fmt = "%s ↛ %s" + annotate_html = ", ".join(short_fmt % (lineno, d) for d in shorts) + annotate_html += " [?]" + + annotate_title = "Line %d was executed, but never jumped to " % lineno + if len(longs) == 1: + annotate_title += longs[0] + elif len(longs) == 2: + annotate_title += longs[0] + " or " + longs[1] + else: + annotate_title += ", ".join(longs[:-1]) + ", or " + longs[-1] elif lineno in analysis.statements: line_class.append(c_run) - # Build the HTML for the line + # Build the HTML for the line. html = [] for tok_type, tok_text in line: if tok_type == "ws": @@ -224,7 +247,7 @@ class HtmlReporter(Reporter): tok_html = escape(tok_text) or ' ' html.append( '<span class="%s">%s</span>' % (tok_type, tok_html) - ) + ) lines.append({ 'html': ''.join(html), @@ -237,13 +260,13 @@ class HtmlReporter(Reporter): # Write the HTML page for this file. template_values = { 'c_exc': c_exc, 'c_mis': c_mis, 'c_par': c_par, 'c_run': c_run, - 'arcs': self.arcs, 'extra_css': self.extra_css, + 'has_arcs': self.has_arcs, 'extra_css': self.extra_css, 'fr': fr, 'nums': nums, 'lines': lines, 'time_stamp': self.time_stamp, } html = spaceless(self.source_tmpl.render(template_values)) - html_filename = flat_rootname + ".html" + html_filename = rootname + ".html" html_path = os.path.join(self.directory, html_filename) self.write_html(html_path, html) @@ -251,31 +274,26 @@ class HtmlReporter(Reporter): index_info = { 'nums': nums, 'html_filename': html_filename, - 'name': fr.name, - } + 'relative_filename': fr.relative_filename(), + } self.files.append(index_info) - self.status.set_index_info(flat_rootname, index_info) + self.status.set_index_info(rootname, index_info) def index_file(self): """Write the index.html file for this report.""" - index_tmpl = Templite( - data("index.html"), self.template_globals - ) + index_tmpl = Templite(data("index.html"), self.template_globals) self.totals = sum(f['nums'] for f in self.files) html = index_tmpl.render({ - 'arcs': self.arcs, + 'has_arcs': self.has_arcs, 'extra_css': self.extra_css, 'files': self.files, 'totals': self.totals, 'time_stamp': self.time_stamp, }) - self.write_html( - os.path.join(self.directory, "index.html"), - html - ) + self.write_html(os.path.join(self.directory, "index.html"), html) # Write the latest hashes for next time. self.status.write(self.directory) @@ -292,11 +310,11 @@ class HtmlStatus(object): # # { # 'format': 1, - # 'settings': '\x87\x9cc8\x80\xe5\x97\xb16\xfcv\xa2\x8d\x8a\xbb\xcf', + # 'settings': '540ee119c15d52a68a53fe6f0897346d', # 'version': '4.0a1', # 'files': { # 'cogapp___init__': { - # 'hash': '\x99*\x0e\\\x10\x11O\x06WG/gJ\x83\xdd\x99', + # 'hash': 'e45581a5b48f879f301c0f30bf77a50c', # 'index': { # 'html_filename': 'cogapp___init__.html', # 'name': 'cogapp/__init__', @@ -305,7 +323,7 @@ class HtmlStatus(object): # }, # ... # 'cogapp_whiteutils': { - # 'hash': 'o\xfd\x0e+s2="\xb2\x1c\xd6\xa1\xee\x85\x85\xda', + # 'hash': '8504bb427fc488c4176809ded0277d51', # 'index': { # 'html_filename': 'cogapp_whiteutils.html', # 'name': 'cogapp/whiteutils', @@ -361,10 +379,17 @@ class HtmlStatus(object): 'version': coverage.__version__, 'settings': self.settings, 'files': files, - } + } with open(status_file, "w") as fout: json.dump(status, fout) + # Older versions of ShiningPanda look for the old name, status.dat. + # Accomodate them if we are running under Jenkins. + # https://issues.jenkins-ci.org/browse/JENKINS-28428 + if "JENKINS_URL" in os.environ: + with open(os.path.join(directory, "status.dat"), "w") as dat: + dat.write("https://issues.jenkins-ci.org/browse/JENKINS-28428\n") + def settings_hash(self): """Get the hash of the coverage.py settings.""" return self.settings @@ -394,16 +419,18 @@ class HtmlStatus(object): def escape(t): """HTML-escape the text in `t`.""" - return (t - # Convert HTML special chars into HTML entities. - .replace("&", "&").replace("<", "<").replace(">", ">") - .replace("'", "'").replace('"', """) - # Convert runs of spaces: "......" -> " . . ." - .replace(" ", " ") - # To deal with odd-length runs, convert the final pair of spaces - # so that "....." -> " . ." - .replace(" ", " ") - ) + return ( + t + # Convert HTML special chars into HTML entities. + .replace("&", "&").replace("<", "<").replace(">", ">") + .replace("'", "'").replace('"', """) + # Convert runs of spaces: "......" -> " . . ." + .replace(" ", " ") + # To deal with odd-length runs, convert the final pair of spaces + # so that "....." -> " . ." + .replace(" ", " ") + ) + def spaceless(html): """Squeeze out some annoying extra space from an HTML string. @@ -415,6 +442,7 @@ def spaceless(html): html = re.sub(r">\s+<p ", ">\n<p ", html) return html + def pair(ratio): """Format a pair of numbers so JavaScript can read them in an attribute.""" return "%s %s" % ratio diff --git a/coverage/htmlfiles/coverage_html.js b/coverage/htmlfiles/coverage_html.js index 5ef0ece5..bd6a8753 100644 --- a/coverage/htmlfiles/coverage_html.js +++ b/coverage/htmlfiles/coverage_html.js @@ -1,3 +1,6 @@ +// Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +// For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + // Coverage.py HTML report browser code. /*jslint browser: true, sloppy: true, vars: true, plusplus: true, maxerr: 50, indent: 4 */ /*global coverage: true, document, window, $ */ diff --git a/coverage/htmlfiles/index.html b/coverage/htmlfiles/index.html index 1afc57c9..ee2deab0 100644 --- a/coverage/htmlfiles/index.html +++ b/coverage/htmlfiles/index.html @@ -1,3 +1,6 @@ +{# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 #} +{# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt #} + <!DOCTYPE html> <html> <head> @@ -41,7 +44,7 @@ <span class="key">s</span> <span class="key">m</span> <span class="key">x</span> - {% if arcs %} + {% if has_arcs %} <span class="key">b</span> <span class="key">p</span> {% endif %} @@ -59,7 +62,7 @@ <th class="shortkey_s">statements</th> <th class="shortkey_m">missing</th> <th class="shortkey_x">excluded</th> - {% if arcs %} + {% if has_arcs %} <th class="shortkey_b">branches</th> <th class="shortkey_p">partial</th> {% endif %} @@ -73,7 +76,7 @@ <td>{{totals.n_statements}}</td> <td>{{totals.n_missing}}</td> <td>{{totals.n_excluded}}</td> - {% if arcs %} + {% if has_arcs %} <td>{{totals.n_branches}}</td> <td>{{totals.n_partial_branches}}</td> {% endif %} @@ -83,11 +86,11 @@ <tbody> {% for file in files %} <tr class="file"> - <td class="name left"><a href="{{file.html_filename}}">{{file.name}}</a></td> + <td class="name left"><a href="{{file.html_filename}}">{{file.relative_filename}}</a></td> <td>{{file.nums.n_statements}}</td> <td>{{file.nums.n_missing}}</td> <td>{{file.nums.n_excluded}}</td> - {% if arcs %} + {% if has_arcs %} <td>{{file.nums.n_branches}}</td> <td>{{file.nums.n_partial_branches}}</td> {% endif %} diff --git a/coverage/htmlfiles/pyfile.html b/coverage/htmlfiles/pyfile.html index d78ba536..ad7969db 100644 --- a/coverage/htmlfiles/pyfile.html +++ b/coverage/htmlfiles/pyfile.html @@ -1,3 +1,6 @@ +{# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 #} +{# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt #} + <!DOCTYPE html> <html> <head> @@ -5,7 +8,7 @@ {# IE8 rounds line-height incorrectly, and adding this emulateIE7 line makes it right! #} {# http://social.msdn.microsoft.com/Forums/en-US/iewebdevelopment/thread/7684445e-f080-4d8f-8529-132763348e21 #} <meta http-equiv="X-UA-Compatible" content="IE=emulateIE7" /> - <title>Coverage for {{fr.name|escape}}: {{nums.pc_covered_str}}%</title> + <title>Coverage for {{fr.relative_filename|escape}}: {{nums.pc_covered_str}}%</title> <link rel="stylesheet" href="style.css" type="text/css"> {% if extra_css %} <link rel="stylesheet" href="{{ extra_css }}" type="text/css"> @@ -22,7 +25,7 @@ <div id="header"> <div class="content"> - <h1>Coverage for <b>{{fr.name|escape}}</b> : + <h1>Coverage for <b>{{fr.relative_filename|escape}}</b> : <span class="pc_cov">{{nums.pc_covered_str}}%</span> </h1> @@ -34,7 +37,7 @@ <span class="{{c_mis}} shortkey_m button_toggle_mis">{{nums.n_missing}} missing</span> <span class="{{c_exc}} shortkey_x button_toggle_exc">{{nums.n_excluded}} excluded</span> - {% if arcs %} + {% if has_arcs %} <span class="{{c_par}} shortkey_p button_toggle_par">{{nums.n_partial_branches}} partial</span> {% endif %} </h2> diff --git a/coverage/htmlfiles/style.css b/coverage/htmlfiles/style.css index 038335c1..15b08904 100644 --- a/coverage/htmlfiles/style.css +++ b/coverage/htmlfiles/style.css @@ -1,4 +1,7 @@ -/* CSS styles for Coverage. */ +/* Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 */ +/* For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt */ + +/* CSS styles for coverage.py. */ /* Page-wide styles */ html, body, h1, h2, h3, p, table, td, th { margin: 0; @@ -249,7 +252,6 @@ td.text { .text span.annotate { font-family: georgia; - font-style: italic; color: #666; float: right; padding-right: .5em; diff --git a/coverage/misc.py b/coverage/misc.py index d5197ea3..db6298b6 100644 --- a/coverage/misc.py +++ b/coverage/misc.py @@ -1,12 +1,64 @@ -"""Miscellaneous stuff for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Miscellaneous stuff for coverage.py.""" import errno import hashlib import inspect +import locale import os +import sys +import types from coverage import env -from coverage.backward import string_class, to_bytes +from coverage.backward import string_class, to_bytes, unicode_class + +ISOLATED_MODULES = {} + + +def isolate_module(mod): + """Copy a module so that we are isolated from aggressive mocking. + + If a test suite mocks os.path.exists (for example), and then we need to use + it during the test, everything will get tangled up if we use their mock. + Making a copy of the module when we import it will isolate coverage.py from + those complications. + """ + if mod not in ISOLATED_MODULES: + new_mod = types.ModuleType(mod.__name__) + ISOLATED_MODULES[mod] = new_mod + for name in dir(mod): + value = getattr(mod, name) + if isinstance(value, types.ModuleType): + value = isolate_module(value) + setattr(new_mod, name, value) + return ISOLATED_MODULES[mod] + +os = isolate_module(os) + + +# Use PyContracts for assertion testing on parameters and returns, but only if +# we are running our own test suite. +if env.TESTING: + from contracts import contract # pylint: disable=unused-import + from contracts import new_contract + + try: + # Define contract words that PyContract doesn't have. + new_contract('bytes', lambda v: isinstance(v, bytes)) + if env.PY3: + new_contract('unicode', lambda v: isinstance(v, unicode_class)) + except ValueError: + # During meta-coverage, this module is imported twice, and PyContracts + # doesn't like redefining contracts. It's OK. + pass +else: # pragma: not covered + # We aren't using real PyContracts, so just define a no-op decorator as a + # stunt double. + def contract(**unused): + """Dummy no-op implementation of `contract`.""" + return lambda func: func def nice_pair(pair): @@ -56,26 +108,25 @@ def format_lines(statements, lines): return ret -def short_stack(): # pragma: debugging - """Return a string summarizing the call stack.""" - stack = inspect.stack()[:0:-1] - return "\n".join("%30s : %s @%d" % (t[3], t[1], t[2]) for t in stack) - - def expensive(fn): - """A decorator to cache the result of an expensive operation. + """A decorator to indicate that a method shouldn't be called more than once. - Only applies to methods with no arguments. + Normally, this does nothing. During testing, this raises an exception if + called more than once. """ - attr = "_cache_" + fn.__name__ - - def _wrapped(self): - """Inner fn that checks the cache.""" - if not hasattr(self, attr): - setattr(self, attr, fn(self)) - return getattr(self, attr) - return _wrapped + if env.TESTING: + attr = "_once_" + fn.__name__ + + def _wrapped(self): + """Inner function that checks the cache.""" + if hasattr(self, attr): + raise Exception("Shouldn't have called %s more than once" % fn.__name__) + setattr(self, attr, True) + return fn(self) + return _wrapped + else: + return fn def bool_or_none(b): @@ -100,6 +151,18 @@ def file_be_gone(path): raise +def output_encoding(outfile=None): + """Determine the encoding to use for output written to `outfile` or stdout.""" + if outfile is None: + outfile = sys.stdout + encoding = ( + getattr(outfile, "encoding", None) or + getattr(sys.__stdout__, "encoding", None) or + locale.getpreferredencoding() + ) + return encoding + + class Hasher(object): """Hashes Python data into md5.""" def __init__(self): @@ -139,30 +202,6 @@ class Hasher(object): return self.md5.hexdigest() -def overrides(obj, method_name, base_class): - """Does `obj` override the `method_name` it got from `base_class`? - - Determine if `obj` implements the method called `method_name`, which it - inherited from `base_class`. - - Returns a boolean. - - """ - klass = obj.__class__ - klass_func = getattr(klass, method_name) - base_func = getattr(base_class, method_name) - - # Python 2/3 compatibility: Python 2 returns an instancemethod object, the - # function is the .im_func attribute. Python 3 returns a plain function - # object already. - if env.PY2: - klass_func = klass_func.im_func - base_func = base_func.im_func - - return klass_func is not base_func - - -# TODO: abc? def _needs_to_implement(that, func_name): """Helper to raise NotImplementedError in interface stubs.""" if hasattr(that, "_coverage_plugin_name"): @@ -181,7 +220,7 @@ def _needs_to_implement(that, func_name): class CoverageException(Exception): - """An exception specific to Coverage.""" + """An exception specific to coverage.py.""" pass diff --git a/coverage/monkey.py b/coverage/monkey.py index ee84d992..c4ec68c6 100644 --- a/coverage/monkey.py +++ b/coverage/monkey.py @@ -1,4 +1,7 @@ -"""Monkey-patching to make coverage work right in some cases.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Monkey-patching to make coverage.py work right in some cases.""" import multiprocessing import multiprocessing.process @@ -6,7 +9,7 @@ import sys # An attribute that will be set on modules to indicate that they have been # monkey-patched. -MARKER = "_coverage$patched" +PATCHED_MARKER = "_coverage$patched" def patch_multiprocessing(): @@ -16,7 +19,7 @@ def patch_multiprocessing(): This is wildly experimental! """ - if hasattr(multiprocessing, MARKER): + if hasattr(multiprocessing, PATCHED_MARKER): return if sys.version_info >= (3, 4): @@ -29,6 +32,7 @@ def patch_multiprocessing(): class ProcessWithCoverage(klass): """A replacement for multiprocess.Process that starts coverage.""" def _bootstrap(self): + """Wrapper around _bootstrap to start coverage.""" from coverage import Coverage cov = Coverage(data_suffix=True) cov.start() @@ -43,4 +47,4 @@ def patch_multiprocessing(): else: multiprocessing.Process = ProcessWithCoverage - setattr(multiprocessing, MARKER, 1) + setattr(multiprocessing, PATCHED_MARKER, True) diff --git a/coverage/parser.py b/coverage/parser.py index fc751eb2..7b8a60f1 100644 --- a/coverage/parser.py +++ b/coverage/parser.py @@ -1,4 +1,7 @@ -"""Code parsing for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Code parsing for coverage.py.""" import collections import dis @@ -9,31 +12,15 @@ import tokenize from coverage.backward import range # pylint: disable=redefined-builtin from coverage.backward import bytes_to_ints from coverage.bytecode import ByteCodes, CodeObjects -from coverage.misc import nice_pair, expensive, join_regex +from coverage.misc import contract, nice_pair, join_regex from coverage.misc import CoverageException, NoSource, NotPython -from coverage.phystokens import generate_tokens - +from coverage.phystokens import compile_unicode, generate_tokens -class CodeParser(object): - """ - Base class for any code parser. - """ - def translate_lines(self, lines): - return set(lines) - def translate_arcs(self, arcs): - return arcs - - def exit_counts(self): - return {} - - def arcs(self): - return [] - - -class PythonParser(CodeParser): +class PythonParser(object): """Parse code to find executable lines, excluded lines, etc.""" + @contract(text='unicode|None') def __init__(self, text=None, filename=None, exclude=None): """ Source can be provided as `text`, the text itself, or `filename`, from @@ -51,40 +38,47 @@ class PythonParser(CodeParser): except IOError as err: raise NoSource( "No source for code: '%s': %s" % (self.filename, err) - ) - - if self.text: - assert isinstance(self.text, str) - # Scrap the BOM if it exists. - # (Used to do this, but no longer. Not sure what bad will happen - # if we don't do it.) - # if ord(self.text[0]) == 0xfeff: - # self.text = self.text[1:] + ) self.exclude = exclude - self.show_tokens = False - # The text lines of the parsed code. self.lines = self.text.split('\n') - # The line numbers of excluded lines of code. + # The normalized line numbers of the statements in the code. Exclusions + # are taken into account, and statements are adjusted to their first + # lines. + self.statements = set() + + # The normalized line numbers of the excluded lines in the code, + # adjusted to their first lines. self.excluded = set() - # The line numbers of docstring lines. - self.docstrings = set() + # The raw_* attributes are only used in this class, and in + # lab/parser.py to show how this class is working. + + # The line numbers that start statements, as reported by the line + # number table in the bytecode. + self.raw_statements = set() + + # The raw line numbers of excluded lines of code, as marked by pragmas. + self.raw_excluded = set() # The line numbers of class definitions. - self.classdefs = set() + self.raw_classdefs = set() - # A dict mapping line numbers to (lo,hi) for multi-line statements. - self.multiline = {} + # The line numbers of docstring lines. + self.raw_docstrings = set() - # The line numbers that start statements. - self.statement_starts = set() + # Internal detail, used by lab/parser.py. + self.show_tokens = False - # Lazily-created ByteParser + # A dict mapping line numbers to (lo,hi) for multi-line statements. + self._multiline = {} + + # Lazily-created ByteParser and arc data. self._byte_parser = None + self._all_arcs = None @property def byte_parser(self): @@ -111,21 +105,23 @@ class PythonParser(CodeParser): def _raw_parse(self): """Parse the source to find the interesting facts about its lines. - A handful of member fields are updated. + A handful of attributes are updated. """ # Find lines which match an exclusion pattern. if self.exclude: - self.excluded = self.lines_matching(self.exclude) + self.raw_excluded = self.lines_matching(self.exclude) # Tokenize, to find excluded suites, to find docstrings, and to find # multi-line statements. indent = 0 exclude_indent = 0 excluding = False + excluding_decorators = False prev_toktype = token.INDENT first_line = None empty = True + first_on_line = True tokgen = generate_tokens(self.text) for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen: @@ -133,37 +129,49 @@ class PythonParser(CodeParser): print("%10s %5s %-20r %r" % ( tokenize.tok_name.get(toktype, toktype), nice_pair((slineno, elineno)), ttext, ltext - )) + )) if toktype == token.INDENT: indent += 1 elif toktype == token.DEDENT: indent -= 1 - elif toktype == token.NAME and ttext == 'class': - # Class definitions look like branches in the byte code, so - # we need to exclude them. The simplest way is to note the - # lines with the 'class' keyword. - self.classdefs.add(slineno) - elif toktype == token.OP and ttext == ':': - if not excluding and elineno in self.excluded: - # Start excluding a suite. We trigger off of the colon - # token so that the #pragma comment will be recognized on - # the same line as the colon. - exclude_indent = indent - excluding = True + elif toktype == token.NAME: + if ttext == 'class': + # Class definitions look like branches in the byte code, so + # we need to exclude them. The simplest way is to note the + # lines with the 'class' keyword. + self.raw_classdefs.add(slineno) + elif toktype == token.OP: + if ttext == ':': + should_exclude = (elineno in self.raw_excluded) or excluding_decorators + if not excluding and should_exclude: + # Start excluding a suite. We trigger off of the colon + # token so that the #pragma comment will be recognized on + # the same line as the colon. + self.raw_excluded.add(elineno) + exclude_indent = indent + excluding = True + excluding_decorators = False + elif ttext == '@' and first_on_line: + # A decorator. + if elineno in self.raw_excluded: + excluding_decorators = True + if excluding_decorators: + self.raw_excluded.add(elineno) elif toktype == token.STRING and prev_toktype == token.INDENT: # Strings that are first on an indented line are docstrings. # (a trick from trace.py in the stdlib.) This works for # 99.9999% of cases. For the rest (!) see: # http://stackoverflow.com/questions/1769332/x/1769794#1769794 - self.docstrings.update(range(slineno, elineno+1)) + self.raw_docstrings.update(range(slineno, elineno+1)) elif toktype == token.NEWLINE: if first_line is not None and elineno != first_line: # We're at the end of a line, and we've ended on a # different line than the first line of the statement, # so record a multi-line range. for l in range(first_line, elineno+1): - self.multiline[l] = first_line + self._multiline[l] = first_line first_line = None + first_on_line = True if ttext.strip() and toktype != tokenize.COMMENT: # A non-whitespace token. @@ -176,17 +184,18 @@ class PythonParser(CodeParser): if excluding and indent <= exclude_indent: excluding = False if excluding: - self.excluded.add(elineno) + self.raw_excluded.add(elineno) + first_on_line = False prev_toktype = toktype # Find the starts of the executable statements. if not empty: - self.statement_starts.update(self.byte_parser._find_statements()) + self.raw_statements.update(self.byte_parser._find_statements()) def first_line(self, line): """Return the first line number of the statement including `line`.""" - first_line = self.multiline.get(line) + first_line = self._multiline.get(line) if first_line: return first_line else: @@ -202,83 +211,77 @@ class PythonParser(CodeParser): return set(self.first_line(l) for l in lines) def translate_lines(self, lines): + """Implement `FileReporter.translate_lines`.""" return self.first_lines(lines) def translate_arcs(self, arcs): - return [ - (self.first_line(a), self.first_line(b)) - for (a, b) in arcs - ] + """Implement `FileReporter.translate_arcs`.""" + return [(self.first_line(a), self.first_line(b)) for (a, b) in arcs] def parse_source(self): """Parse source text to find executable lines, excluded lines, etc. - Return values are 1) a set of executable line numbers, and 2) a set of - excluded line numbers. - - Reported line numbers are normalized to the first line of multi-line - statements. + Sets the .excluded and .statements attributes, normalized to the first + line of multi-line statements. """ try: self._raw_parse() - except (tokenize.TokenError, IndentationError) as tokerr: - msg, lineno = tokerr.args # pylint: disable=unpacking-non-sequence + except (tokenize.TokenError, IndentationError) as err: + if hasattr(err, "lineno"): + lineno = err.lineno # IndentationError + else: + lineno = err.args[1][0] # TokenError raise NotPython( - "Couldn't parse '%s' as Python source: '%s' at %s" % - (self.filename, msg, lineno) + u"Couldn't parse '%s' as Python source: '%s' at line %d" % ( + self.filename, err.args[0], lineno ) + ) - excluded_lines = self.first_lines(self.excluded) - ignore = set() - ignore.update(excluded_lines) - ignore.update(self.docstrings) - starts = self.statement_starts - ignore - lines = self.first_lines(starts) - lines -= ignore + self.excluded = self.first_lines(self.raw_excluded) - return lines, excluded_lines + ignore = self.excluded | self.raw_docstrings + starts = self.raw_statements - ignore + self.statements = self.first_lines(starts) - ignore - @expensive def arcs(self): """Get information about the arcs available in the code. - Returns a sorted list of line number pairs. Line numbers have been - normalized to the first line of multi-line statements. + Returns a set of line number pairs. Line numbers have been normalized + to the first line of multi-line statements. """ - all_arcs = [] - for l1, l2 in self.byte_parser._all_arcs(): - fl1 = self.first_line(l1) - fl2 = self.first_line(l2) - if fl1 != fl2: - all_arcs.append((fl1, fl2)) - return sorted(all_arcs) - - @expensive + if self._all_arcs is None: + self._all_arcs = set() + for l1, l2 in self.byte_parser._all_arcs(): + fl1 = self.first_line(l1) + fl2 = self.first_line(l2) + if fl1 != fl2: + self._all_arcs.add((fl1, fl2)) + return self._all_arcs + def exit_counts(self): - """Get a mapping from line numbers to count of exits from that line. + """Get a count of exits from that each line. Excluded lines are excluded. """ - excluded_lines = self.first_lines(self.excluded) exit_counts = collections.defaultdict(int) for l1, l2 in self.arcs(): if l1 < 0: # Don't ever report -1 as a line number continue - if l1 in excluded_lines: + if l1 in self.excluded: # Don't report excluded lines as line numbers. continue - if l2 in excluded_lines: + if l2 in self.excluded: # Arcs to excluded lines shouldn't count. continue exit_counts[l1] += 1 # Class definitions have one extra exit, so remove one for each: - for l in self.classdefs: - # Ensure key is there: classdefs can include excluded lines. + for l in self.raw_classdefs: + # Ensure key is there: class definitions can include excluded lines. if l in exit_counts: exit_counts[l] -= 1 @@ -309,7 +312,7 @@ OPS_CODE_END = _opcode_set('RETURN_VALUE') OPS_CHUNK_END = _opcode_set( 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'RETURN_VALUE', 'RAISE_VARARGS', 'BREAK_LOOP', 'CONTINUE_LOOP', - ) +) # Opcodes that unconditionally begin a new code chunk. By starting new chunks # with unconditional jump instructions, we neatly deal with jumps to jumps @@ -319,7 +322,7 @@ OPS_CHUNK_BEGIN = _opcode_set('JUMP_ABSOLUTE', 'JUMP_FORWARD') # Opcodes that push a block on the block stack. OPS_PUSH_BLOCK = _opcode_set( 'SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH' - ) +) # Block types for exception handling. OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY') @@ -334,7 +337,7 @@ OPS_NO_JUMP = OPS_PUSH_BLOCK OP_BREAK_LOOP = _opcode('BREAK_LOOP') OP_END_FINALLY = _opcode('END_FINALLY') OP_COMPARE_OP = _opcode('COMPARE_OP') -COMPARE_EXCEPTION = 10 # just have to get this const from the code. +COMPARE_EXCEPTION = 10 # just have to get this constant from the code. OP_LOAD_CONST = _opcode('LOAD_CONST') OP_RETURN_VALUE = _opcode('RETURN_VALUE') @@ -342,16 +345,17 @@ OP_RETURN_VALUE = _opcode('RETURN_VALUE') class ByteParser(object): """Parse byte codes to understand the structure of code.""" + @contract(text='unicode') def __init__(self, text, code=None, filename=None): self.text = text if code: self.code = code else: try: - self.code = compile(text, filename, "exec") + self.code = compile_unicode(text, filename, "exec") except SyntaxError as synerr: raise NotPython( - "Couldn't parse '%s' as Python source: '%s' at line %d" % ( + u"Couldn't parse '%s' as Python source: '%s' at line %d" % ( filename, synerr.msg, synerr.lineno ) ) @@ -361,10 +365,9 @@ class ByteParser(object): for attr in ['co_lnotab', 'co_firstlineno', 'co_consts', 'co_code']: if not hasattr(self.code, attr): raise CoverageException( - "This implementation of Python doesn't support code " - "analysis.\n" + "This implementation of Python doesn't support code analysis.\n" "Run coverage.py under CPython for this command." - ) + ) def child_parsers(self): """Iterate over all the code objects nested within this one. @@ -682,4 +685,4 @@ class Chunk(object): "!" if self.first else "", "v" if self.entrance else "", list(self.exits), - ) + ) diff --git a/coverage/phystokens.py b/coverage/phystokens.py index ed6bd238..b34b1c3b 100644 --- a/coverage/phystokens.py +++ b/coverage/phystokens.py @@ -1,13 +1,18 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Better tokenizing for coverage.py.""" import codecs import keyword import re +import sys import token import tokenize from coverage import env from coverage.backward import iternext +from coverage.misc import contract def phys_tokens(toks): @@ -66,6 +71,7 @@ def phys_tokens(toks): last_lineno = elineno +@contract(source='unicode') def source_token_lines(source): """Generate a series of lines, one for each line in `source`. @@ -104,7 +110,7 @@ def source_token_lines(source): mark_end = False else: if mark_start and scol > col: - line.append(("ws", " " * (scol - col))) + line.append(("ws", u" " * (scol - col))) mark_start = False tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3] if ttype == token.NAME and keyword.iskeyword(ttext): @@ -134,11 +140,10 @@ class CachedTokenizer(object): self.last_text = None self.last_tokens = None + @contract(text='unicode') def generate_tokens(self, text): """A stand-in for `tokenize.generate_tokens`.""" - # Check the type first so we don't compare bytes to unicode and get - # warnings. - if type(text) != type(self.last_text) or text != self.last_text: + if text != self.last_text: self.last_text = text readline = iternext(text.splitlines(True)) self.last_tokens = list(tokenize.generate_tokens(readline)) @@ -148,14 +153,15 @@ class CachedTokenizer(object): generate_tokens = CachedTokenizer().generate_tokens +COOKIE_RE = re.compile(r"^[ \t]*#.*coding[:=][ \t]*([-\w.]+)", flags=re.MULTILINE) + +@contract(source='bytes') def _source_encoding_py2(source): """Determine the encoding for `source`, according to PEP 263. - Arguments: - source (byte string): the text of the program. + `source` is a byte string, the text of the program. - Returns: - string: the name of the encoding. + Returns a string, the name of the encoding. """ assert isinstance(source, bytes) @@ -165,8 +171,6 @@ def _source_encoding_py2(source): # This is mostly code adapted from Py3.2's tokenize module. - cookie_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)") - def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" # Only care about the first 12 characters. @@ -204,7 +208,7 @@ def _source_encoding_py2(source): except UnicodeDecodeError: return None - matches = cookie_re.findall(line_string) + matches = COOKIE_RE.findall(line_string) if not matches: return None encoding = _get_normal_name(matches[0]) @@ -246,17 +250,15 @@ def _source_encoding_py2(source): return default +@contract(source='bytes') def _source_encoding_py3(source): """Determine the encoding for `source`, according to PEP 263. - Arguments: - source (byte string): the text of the program. + `source` is a byte string: the text of the program. - Returns: - string: the name of the encoding. + Returns a string, the name of the encoding. """ - assert isinstance(source, bytes) readline = iternext(source.splitlines(True)) return tokenize.detect_encoding(readline)[0] @@ -265,3 +267,29 @@ if env.PY3: source_encoding = _source_encoding_py3 else: source_encoding = _source_encoding_py2 + + +@contract(source='unicode') +def compile_unicode(source, filename, mode): + """Just like the `compile` builtin, but works on any Unicode string. + + Python 2's compile() builtin has a stupid restriction: if the source string + is Unicode, then it may not have a encoding declaration in it. Why not? + Who knows! It also decodes to utf8, and then tries to interpret those utf8 + bytes according to the encoding declaration. Why? Who knows! + + This function neuters the coding declaration, and compiles it. + + """ + source = neuter_encoding_declaration(source) + if env.PY2 and isinstance(filename, unicode): + filename = filename.encode(sys.getfilesystemencoding(), "replace") + code = compile(source, filename, mode) + return code + + +@contract(source='unicode', returns='unicode') +def neuter_encoding_declaration(source): + """Return `source`, with any encoding declaration neutered.""" + source = COOKIE_RE.sub("# (deleted declaration)", source, count=1) + return source diff --git a/coverage/pickle2json.py b/coverage/pickle2json.py new file mode 100644 index 00000000..95b42ef3 --- /dev/null +++ b/coverage/pickle2json.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Convert pickle to JSON for coverage.py.""" + +from coverage.backward import pickle +from coverage.data import CoverageData + + +def pickle_read_raw_data(cls_unused, file_obj): + """Replacement for CoverageData._read_raw_data.""" + return pickle.load(file_obj) + + +def pickle2json(infile, outfile): + """Convert a coverage.py 3.x pickle data file to a 4.x JSON data file.""" + try: + old_read_raw_data = CoverageData._read_raw_data + CoverageData._read_raw_data = pickle_read_raw_data + + covdata = CoverageData() + + with open(infile, 'rb') as inf: + covdata.read_fileobj(inf) + + covdata.write_file(outfile) + finally: + CoverageData._read_raw_data = old_read_raw_data + + +if __name__ == "__main__": + from optparse import OptionParser + + parser = OptionParser(usage="usage: %s [options]" % __file__) + parser.description = "Convert .coverage files from pickle to JSON format" + parser.add_option( + "-i", "--input-file", action="store", default=".coverage", + help="Name of input file. Default .coverage", + ) + parser.add_option( + "-o", "--output-file", action="store", default=".coverage", + help="Name of output file. Default .coverage", + ) + + (options, args) = parser.parse_args() + + pickle2json(options.input_file, options.output_file) diff --git a/coverage/plugin.py b/coverage/plugin.py index 6648d7a6..f870c254 100644 --- a/coverage/plugin.py +++ b/coverage/plugin.py @@ -1,19 +1,18 @@ -"""Plugin interfaces for coverage.py""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt -import os -import re +"""Plugin interfaces for coverage.py""" -from coverage.misc import _needs_to_implement +from coverage import files +from coverage.misc import contract, _needs_to_implement -# TODO: document that the plugin objects may be decorated with attributes with -# named "_coverage_*". class CoveragePlugin(object): """Base class for coverage.py plugins. - To write a coverage.py plugin, create a subclass of `CoveragePlugin`. - You can override methods here to participate in various aspects of - coverage.py's processing. + To write a coverage.py plugin, create a module with a subclass of + :class:`CoveragePlugin`. You will override methods in your class to + participate in various aspects of coverage.py's processing. Currently the only plugin type is a file tracer, for implementing measurement support for non-Python files. File tracer plugins implement @@ -23,122 +22,149 @@ class CoveragePlugin(object): Any plugin can optionally implement :meth:`sys_info` to provide debugging information about their operation. - """ + Coverage.py will store its own information on your plugin object, using + attributes whose names start with ``_coverage_``. Don't be startled. - def __init__(self, options): - """ - When the plugin is constructed, it will be passed a dictionary of - plugin-specific options read from the .coveragerc configuration file. - The base class stores these on the `self.options` attribute. + To register your plugin, define a function called `coverage_init` in your + module:: - Arguments: - options (dict): The plugin-specific options read from the - .coveragerc configuration file. + def coverage_init(reg, options): + reg.add_file_tracer(MyPlugin()) - """ - self.options = options + You use the `reg` parameter passed to your `coverage_init` function to + register your plugin object. It has one method, `add_file_tracer`, which + takes a newly created instance of your plugin. + + If your plugin takes options, the `options` parameter is a dictionary of + your plugin's options from the coverage.py configuration file. Use them + however you want to configure your object before registering it. + + """ def file_tracer(self, filename): # pylint: disable=unused-argument - """Return a FileTracer object for a file. + """Get a :class:`FileTracer` object for a file. - Every source file is offered to the plugin to give it a chance to take - responsibility for tracing the file. If your plugin can handle the - file, then return a :class:`FileTracer` object. Otherwise return None. + Every Python source file is offered to the plugin to give it a chance + to take responsibility for tracing the file. If your plugin can handle + the file, then return a :class:`FileTracer` object. Otherwise return + None. There is no way to register your plugin for particular files. Instead, - this method is invoked for all files, and can decide whether it can - trace the file or not. Be prepared for `filename` to refer to all + this method is invoked for all files, and the plugin decides whether it + can trace the file or not. Be prepared for `filename` to refer to all kinds of files that have nothing to do with your plugin. - Arguments: - filename (str): The path to the file being considered. This is the - absolute real path to the file. If you are comparing to other - paths, be sure to take this into account. + The file name will be a Python file being executed. There are two + broad categories of behavior for a plugin, depending on the kind of + files your plugin supports: + + * Static file names: each of your original source files has been + converted into a distinct Python file. Your plugin is invoked with + the Python file name, and it maps it back to its original source + file. + + * Dynamic file names: all of your source files are executed by the same + Python file. In this case, your plugin implements + :meth:`FileTracer.dynamic_source_filename` to provide the actual + source file for each execution frame. + + `filename` is a string, the path to the file being considered. This is + the absolute real path to the file. If you are comparing to other + paths, be sure to take this into account. - Returns: - FileTracer: the :class:`FileTracer` object to use to trace - `filename`, or None if this plugin cannot trace this file. + Returns a :class:`FileTracer` object to use to trace `filename`, or + None if this plugin cannot trace this file. """ return None def file_reporter(self, filename): # pylint: disable=unused-argument - """Return the FileReporter class to use for filename. + """Get the :class:`FileReporter` class to use for a file. This will only be invoked if `filename` returns non-None from - :meth:`file_tracer`. It's an error to return None. + :meth:`file_tracer`. It's an error to return None from this method. + + Returns a :class:`FileReporter` object to use to report on `filename`. """ _needs_to_implement(self, "file_reporter") def sys_info(self): - """Return a list of information useful for debugging. + """Get a list of information useful for debugging. This method will be invoked for ``--debug=sys``. Your plugin can return any information it wants to be displayed. - The return value is a list of pairs: (name, value). + Returns a list of pairs: `[(name, value), ...]`. """ return [] class FileTracer(object): - """Support needed for files during the tracing phase. + """Support needed for files during the execution phase. You may construct this object from :meth:`CoveragePlugin.file_tracer` any - way you like. A natural choice would be to pass the filename given to + way you like. A natural choice would be to pass the file name given to `file_tracer`. + `FileTracer` objects should only be created in the + :meth:`CoveragePlugin.file_tracer` method. + + See :ref:`howitworks` for details of the different coverage.py phases. + """ def source_filename(self): - """The source filename for this file. + """The source file name for this file. - This may be any filename you like. A key responsibility of a plugin is - to own the mapping from Python execution back to whatever source - filename was originally the source of the code. + This may be any file name you like. A key responsibility of a plugin + is to own the mapping from Python execution back to whatever source + file name was originally the source of the code. - Returns: - The filename to credit with this execution. + See :meth:`CoveragePlugin.file_tracer` for details about static and + dynamic file names. + + Returns the file name to credit with this execution. """ _needs_to_implement(self, "source_filename") def has_dynamic_source_filename(self): - """Does this FileTracer have dynamic source filenames? + """Does this FileTracer have dynamic source file names? - FileTracers can provide dynamically determined filenames by - implementing dynamic_source_filename. Invoking that function is - expensive. To determine whether to invoke it, coverage.py uses - the result of this function to know if it needs to bother invoking + FileTracers can provide dynamically determined file names by + implementing :meth:`dynamic_source_filename`. Invoking that function + is expensive. To determine whether to invoke it, coverage.py uses the + result of this function to know if it needs to bother invoking :meth:`dynamic_source_filename`. - Returns: - boolean: True if :meth:`dynamic_source_filename` should be called - to get dynamic source filenames. + See :meth:`CoveragePlugin.file_tracer` for details about static and + dynamic file names. + + Returns True if :meth:`dynamic_source_filename` should be called to get + dynamic source file names. """ return False def dynamic_source_filename(self, filename, frame): # pylint: disable=unused-argument - """Returns a dynamically computed source filename. + """Get a dynamically computed source file name. - Some plugins need to compute the source filename dynamically for each + Some plugins need to compute the source file name dynamically for each frame. This function will not be invoked if :meth:`has_dynamic_source_filename` returns False. - Returns: - The source filename for this frame, or None if this frame shouldn't - be measured. + Returns the source file name for this frame, or None if this frame + shouldn't be measured. """ return None def line_number_range(self, frame): - """Return the range of source line numbers for a given a call frame. + """Get the range of source line numbers for a given a call frame. The call frame is examined, and the source line number in the original file is returned. The return value is a pair of numbers, the starting @@ -150,103 +176,206 @@ class FileTracer(object): from the source file were executed. Return (-1, -1) in this case to tell coverage.py that no lines should be recorded for this frame. - Arguments: - frame: the call frame to examine. - - Returns: - int, int: a pair of line numbers, the start and end lines - executed in the source, inclusive. - """ lineno = frame.f_lineno return lineno, lineno class FileReporter(object): - """Support needed for files during the reporting phase.""" + """Support needed for files during the analysis and reporting phases. + + See :ref:`howitworks` for details of the different coverage.py phases. + + `FileReporter` objects should only be created in the + :meth:`CoveragePlugin.file_reporter` method. + + There are many methods here, but only :meth:`lines` is required, to provide + the set of executable lines in the file. + + """ + def __init__(self, filename): - # TODO: document that this init happens. + """Simple initialization of a `FileReporter`. + + The `filename` argument is the path to the file being reported. This + will be available as the `.filename` attribute on the object. Other + method implementations on this base class rely on this attribute. + + """ self.filename = filename def __repr__(self): - return ( - # pylint: disable=redundant-keyword-arg - "<{self.__class__.__name__}" - " filename={self.filename!r}>".format(self=self) - ) + return "<{0.__class__.__name__} filename={0.filename!r}>".format(self) - # Annoying comparison operators. Py3k wants __lt__ etc, and Py2k needs all - # of them defined. + def relative_filename(self): + """Get the relative file name for this file. - def __lt__(self, other): - return self.filename < other.filename + This file path will be displayed in reports. The default + implementation will supply the actual project-relative file path. You + only need to supply this method if you have an unusual syntax for file + paths. - def __le__(self, other): - return self.filename <= other.filename + """ + return files.relative_filename(self.filename) - def __eq__(self, other): - return self.filename == other.filename + @contract(returns='unicode') + def source(self): + """Get the source for the file. - def __ne__(self, other): - return self.filename != other.filename + Returns a Unicode string. - def __gt__(self, other): - return self.filename > other.filename + The base implementation simply reads the `self.filename` file and + decodes it as UTF8. Override this method if your file isn't readable + as a text file, or if you need other encoding support. - def __ge__(self, other): - return self.filename >= other.filename + """ + with open(self.filename, "rb") as f: + return f.read().decode("utf8") - def statements(self): - _needs_to_implement(self, "statements") + def lines(self): + """Get the executable lines in this file. - def excluded_statements(self): - return set([]) + Your plugin must determine which lines in the file were possibly + executable. This method returns a set of those line numbers. + + Returns a set of line numbers. + + """ + _needs_to_implement(self, "lines") + + def excluded_lines(self): + """Get the excluded executable lines in this file. + + Your plugin can use any method it likes to allow the user to exclude + executable lines from consideration. + + Returns a set of line numbers. + + The base implementation returns the empty set. + + """ + return set() def translate_lines(self, lines): + """Translate recorded lines into reported lines. + + Some file formats will want to report lines slightly differently than + they are recorded. For example, Python records the last line of a + multi-line statement, but reports are nicer if they mention the first + line. + + Your plugin can optionally define this method to perform these kinds of + adjustment. + + `lines` is a sequence of integers, the recorded line numbers. + + Returns a set of integers, the adjusted line numbers. + + The base implementation returns the numbers unchanged. + + """ return set(lines) - def translate_arcs(self, arcs): - return arcs + def arcs(self): + """Get the executable arcs in this file. + + To support branch coverage, your plugin needs to be able to indicate + possible execution paths, as a set of line number pairs. Each pair is + a `(prev, next)` pair indicating that execution can transition from the + `prev` line number to the `next` line number. + + Returns a set of pairs of line numbers. The default implementation + returns an empty set. + + """ + return set() def no_branch_lines(self): + """Get the lines excused from branch coverage in this file. + + Your plugin can use any method it likes to allow the user to exclude + lines from consideration of branch coverage. + + Returns a set of line numbers. + + The base implementation returns the empty set. + + """ return set() + def translate_arcs(self, arcs): + """Translate recorded arcs into reported arcs. + + Similar to :meth:`translate_lines`, but for arcs. `arcs` is a set of + line number pairs. + + Returns a set of line number pairs. + + The default implementation returns `arcs` unchanged. + + """ + return arcs + def exit_counts(self): - return {} + """Get a count of exits from that each line. - def arcs(self): - return [] + To determine which lines are branches, coverage.py looks for lines that + have more than one exit. This function creates a dict mapping each + executable line number to a count of how many exits it has. - def source(self): - """Return the source for the code, a Unicode string.""" - # A generic implementation: read the text of self.filename - with open(self.filename) as f: - return f.read() + To be honest, this feels wrong, and should be refactored. Let me know + if you attempt to implement this... + + """ + return {} def source_token_lines(self): - """Return the 'tokenized' text for the code.""" - # A generic implementation, each line is one "txt" token. + """Generate a series of tokenized lines, one for each line in `source`. + + These tokens are used for syntax-colored reports. + + Each line is a list of pairs, each pair is a token:: + + [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ] + + Each pair has a token class, and the token text. The token classes + are: + + * ``'com'``: a comment + * ``'key'``: a keyword + * ``'nam'``: a name, or identifier + * ``'num'``: a number + * ``'op'``: an operator + * ``'str'``: a string literal + * ``'txt'``: some other kind of text + + If you concatenate all the token texts, and then join them with + newlines, you should have your original source back. + + The default implementation simply returns each line tagged as + ``'txt'``. + + """ for line in self.source().splitlines(): yield [('txt', line)] - def should_be_python(self): - """Does it seem like this file should contain Python? + # Annoying comparison operators. Py3k wants __lt__ etc, and Py2k needs all + # of them defined. + + def __eq__(self, other): + return isinstance(other, FileReporter) and self.filename == other.filename - This is used to decide if a file reported as part of the execution of - a program was really likely to have contained Python in the first - place. - """ - return False + def __ne__(self, other): + return not (self == other) - def flat_rootname(self): - """A base for a flat filename to correspond to this file. + def __lt__(self, other): + return self.filename < other.filename - Useful for writing files about the code where you want all the files in - the same directory, but need to differentiate same-named files from - different directories. + def __le__(self, other): + return self.filename <= other.filename - For example, the file a/b/c.py will return 'a_b_c_py' + def __gt__(self, other): + return self.filename > other.filename - """ - name = os.path.splitdrive(self.name)[1] - return re.sub(r"[\\/.:]", "_", name) + def __ge__(self, other): + return self.filename >= other.filename diff --git a/coverage/plugin_support.py b/coverage/plugin_support.py new file mode 100644 index 00000000..8a4fbec5 --- /dev/null +++ b/coverage/plugin_support.py @@ -0,0 +1,247 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Support for plugins.""" + +import os +import os.path +import sys + +from coverage.misc import CoverageException, isolate_module +from coverage.plugin import CoveragePlugin, FileTracer, FileReporter + +os = isolate_module(os) + + +class Plugins(object): + """The currently loaded collection of coverage.py plugins.""" + + def __init__(self): + self.order = [] + self.names = {} + self.file_tracers = [] + + self.current_module = None + self.debug = None + + @classmethod + def load_plugins(cls, modules, config, debug=None): + """Load plugins from `modules`. + + Returns a list of loaded and configured plugins. + + """ + plugins = cls() + plugins.debug = debug + + for module in modules: + plugins.current_module = module + __import__(module) + mod = sys.modules[module] + + coverage_init = getattr(mod, "coverage_init", None) + if not coverage_init: + raise CoverageException( + "Plugin module %r didn't define a coverage_init function" % module + ) + + options = config.get_plugin_options(module) + coverage_init(plugins, options) + + plugins.current_module = None + return plugins + + def add_file_tracer(self, plugin): + """Add a file tracer plugin. + + `plugin` is an instance of a third-party plugin class. It must + implement the :meth:`CoveragePlugin.file_tracer` method. + + """ + self._add_plugin(plugin, self.file_tracers) + + def add_noop(self, plugin): + """Add a plugin that does nothing. + + This is only useful for testing the plugin support. + + """ + self._add_plugin(plugin, None) + + def _add_plugin(self, plugin, specialized): + """Add a plugin object. + + `plugin` is a :class:`CoveragePlugin` instance to add. `specialized` + is a list to append the plugin to. + + """ + plugin_name = "%s.%s" % (self.current_module, plugin.__class__.__name__) + if self.debug and self.debug.should('plugin'): + self.debug.write("Loaded plugin %r: %r" % (self.current_module, plugin)) + labelled = LabelledDebug("plugin %r" % (self.current_module,), self.debug) + plugin = DebugPluginWrapper(plugin, labelled) + + # pylint: disable=attribute-defined-outside-init + plugin._coverage_plugin_name = plugin_name + plugin._coverage_enabled = True + self.order.append(plugin) + self.names[plugin_name] = plugin + if specialized is not None: + specialized.append(plugin) + + def __nonzero__(self): + return bool(self.order) + + __bool__ = __nonzero__ + + def __iter__(self): + return iter(self.order) + + def get(self, plugin_name): + """Return a plugin by name.""" + return self.names[plugin_name] + + +class LabelledDebug(object): + """A Debug writer, but with labels for prepending to the messages.""" + + def __init__(self, label, debug, prev_labels=()): + self.labels = list(prev_labels) + [label] + self.debug = debug + + def add_label(self, label): + """Add a label to the writer, and return a new `LabelledDebug`.""" + return LabelledDebug(label, self.debug, self.labels) + + def message_prefix(self): + """The prefix to use on messages, combining the labels.""" + prefixes = self.labels + [''] + return ":\n".join(" "*i+label for i, label in enumerate(prefixes)) + + def write(self, message): + """Write `message`, but with the labels prepended.""" + self.debug.write("%s%s" % (self.message_prefix(), message)) + + +class DebugPluginWrapper(CoveragePlugin): + """Wrap a plugin, and use debug to report on what it's doing.""" + + def __init__(self, plugin, debug): + super(DebugPluginWrapper, self).__init__() + self.plugin = plugin + self.debug = debug + + def file_tracer(self, filename): + tracer = self.plugin.file_tracer(filename) + self.debug.write("file_tracer(%r) --> %r" % (filename, tracer)) + if tracer: + debug = self.debug.add_label("file %r" % (filename,)) + tracer = DebugFileTracerWrapper(tracer, debug) + return tracer + + def file_reporter(self, filename): + reporter = self.plugin.file_reporter(filename) + self.debug.write("file_reporter(%r) --> %r" % (filename, reporter)) + if reporter: + debug = self.debug.add_label("file %r" % (filename,)) + reporter = DebugFileReporterWrapper(filename, reporter, debug) + return reporter + + def sys_info(self): + return self.plugin.sys_info() + + +class DebugFileTracerWrapper(FileTracer): + """A debugging `FileTracer`.""" + + def __init__(self, tracer, debug): + self.tracer = tracer + self.debug = debug + + def _show_frame(self, frame): + """A short string identifying a frame, for debug messages.""" + return "%s@%d" % ( + os.path.basename(frame.f_code.co_filename), + frame.f_lineno, + ) + + def source_filename(self): + sfilename = self.tracer.source_filename() + self.debug.write("source_filename() --> %r" % (sfilename,)) + return sfilename + + def has_dynamic_source_filename(self): + has = self.tracer.has_dynamic_source_filename() + self.debug.write("has_dynamic_source_filename() --> %r" % (has,)) + return has + + def dynamic_source_filename(self, filename, frame): + dyn = self.tracer.dynamic_source_filename(filename, frame) + self.debug.write("dynamic_source_filename(%r, %s) --> %r" % ( + filename, self._show_frame(frame), dyn, + )) + return dyn + + def line_number_range(self, frame): + pair = self.tracer.line_number_range(frame) + self.debug.write("line_number_range(%s) --> %r" % (self._show_frame(frame), pair)) + return pair + + +class DebugFileReporterWrapper(FileReporter): + """A debugging `FileReporter`.""" + + def __init__(self, filename, reporter, debug): + super(DebugFileReporterWrapper, self).__init__(filename) + self.reporter = reporter + self.debug = debug + + def relative_filename(self): + ret = self.reporter.relative_filename() + self.debug.write("relative_filename() --> %r" % (ret,)) + return ret + + def lines(self): + ret = self.reporter.lines() + self.debug.write("lines() --> %r" % (ret,)) + return ret + + def excluded_lines(self): + ret = self.reporter.excluded_lines() + self.debug.write("excluded_lines() --> %r" % (ret,)) + return ret + + def translate_lines(self, lines): + ret = self.reporter.translate_lines(lines) + self.debug.write("translate_lines(%r) --> %r" % (lines, ret)) + return ret + + def translate_arcs(self, arcs): + ret = self.reporter.translate_arcs(arcs) + self.debug.write("translate_arcs(%r) --> %r" % (arcs, ret)) + return ret + + def no_branch_lines(self): + ret = self.reporter.no_branch_lines() + self.debug.write("no_branch_lines() --> %r" % (ret,)) + return ret + + def exit_counts(self): + ret = self.reporter.exit_counts() + self.debug.write("exit_counts() --> %r" % (ret,)) + return ret + + def arcs(self): + ret = self.reporter.arcs() + self.debug.write("arcs() --> %r" % (ret,)) + return ret + + def source(self): + ret = self.reporter.source() + self.debug.write("source() --> %d chars" % (len(ret),)) + return ret + + def source_token_lines(self): + ret = list(self.reporter.source_token_lines()) + self.debug.write("source_token_lines() --> %d tokens" % (len(ret),)) + return ret diff --git a/coverage/python.py b/coverage/python.py index 19212a5b..5e563828 100644 --- a/coverage/python.py +++ b/coverage/python.py @@ -1,37 +1,34 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Python source expertise for coverage.py""" import os.path -import sys -import tokenize import zipimport -from coverage import env -from coverage.backward import unicode_class -from coverage.files import FileLocator -from coverage.misc import NoSource, join_regex +from coverage import env, files +from coverage.misc import contract, expensive, NoSource, join_regex, isolate_module from coverage.parser import PythonParser from coverage.phystokens import source_token_lines, source_encoding from coverage.plugin import FileReporter +os = isolate_module(os) + +@contract(returns='bytes') def read_python_source(filename): """Read the Python source text from `filename`. - Returns a str: unicode on Python 3, bytes on Python 2. + Returns bytes. """ - # Python 3.2 provides `tokenize.open`, the best way to open source files. - if sys.version_info >= (3, 2): - f = tokenize.open(filename) - else: - f = open(filename, "rU") - - with f: - return f.read() + with open(filename, "rb") as f: + return f.read().replace(b"\r\n", b"\n").replace(b"\r", b"\n") +@contract(returns='unicode') def get_python_source(filename): - """Return the source code, as a str.""" + """Return the source code, as unicode.""" base, ext = os.path.splitext(filename) if ext == ".py" and env.WINDOWS: exts = [".py", ".pyw"] @@ -48,13 +45,13 @@ def get_python_source(filename): # Maybe it's in a zip file? source = get_zip_bytes(try_filename) if source is not None: - if env.PY3: - source = source.decode(source_encoding(source)) break else: # Couldn't find source. raise NoSource("No source for code: '%s'." % filename) + source = source.decode(source_encoding(source), "replace") + # Python code should always end with a line with a newline. if source and source[-1] != '\n': source += '\n' @@ -62,6 +59,7 @@ def get_python_source(filename): return source +@contract(returns='bytes|None') def get_zip_bytes(filename): """Get data from `filename` if it is a zip file path. @@ -82,7 +80,6 @@ def get_zip_bytes(filename): data = zi.get_data(parts[1]) except IOError: continue - assert isinstance(data, bytes) return data return None @@ -92,55 +89,57 @@ class PythonFileReporter(FileReporter): def __init__(self, morf, coverage=None): self.coverage = coverage - file_locator = coverage.file_locator if coverage else FileLocator() if hasattr(morf, '__file__'): filename = morf.__file__ else: filename = morf + filename = files.unicode_filename(filename) + # .pyc files should always refer to a .py instead. if filename.endswith(('.pyc', '.pyo')): filename = filename[:-1] elif filename.endswith('$py.class'): # Jython filename = filename[:-9] + ".py" - super(PythonFileReporter, self).__init__( - file_locator.canonical_filename(filename) - ) + super(PythonFileReporter, self).__init__(files.canonical_filename(filename)) if hasattr(morf, '__name__'): name = morf.__name__ name = name.replace(".", os.sep) + ".py" + name = files.unicode_filename(name) else: - name = file_locator.relative_filename(filename) - self.name = name + name = files.relative_filename(filename) + self.relname = name self._source = None self._parser = None self._statements = None self._excluded = None + @contract(returns='unicode') + def relative_filename(self): + return self.relname + @property def parser(self): + """Lazily create a :class:`PythonParser`.""" if self._parser is None: self._parser = PythonParser( filename=self.filename, exclude=self.coverage._exclude_regex('exclude'), ) + self._parser.parse_source() return self._parser - def statements(self): + def lines(self): """Return the line numbers of statements in the file.""" - if self._statements is None: - self._statements, self._excluded = self.parser.parse_source() - return self._statements + return self.parser.statements - def excluded_statements(self): + def excluded_lines(self): """Return the line numbers of statements in the file.""" - if self._excluded is None: - self._statements, self._excluded = self.parser.parse_source() - return self._excluded + return self.parser.excluded def translate_lines(self, lines): return self.parser.translate_lines(lines) @@ -148,6 +147,7 @@ class PythonFileReporter(FileReporter): def translate_arcs(self, arcs): return self.parser.translate_arcs(arcs) + @expensive def no_branch_lines(self): no_branch = self.parser.lines_matching( join_regex(self.coverage.config.partial_list), @@ -155,19 +155,18 @@ class PythonFileReporter(FileReporter): ) return no_branch + @expensive def arcs(self): return self.parser.arcs() + @expensive def exit_counts(self): return self.parser.exit_counts() + @contract(returns='unicode') def source(self): if self._source is None: self._source = get_python_source(self.filename) - if env.PY2: - encoding = source_encoding(self._source) - self._source = self._source.decode(encoding, "replace") - assert isinstance(self._source, unicode_class) return self._source def should_be_python(self): diff --git a/coverage/pytracer.py b/coverage/pytracer.py index 3f03aaf7..cdb3ae70 100644 --- a/coverage/pytracer.py +++ b/coverage/pytracer.py @@ -1,4 +1,7 @@ -"""Raw data collector for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Raw data collector for coverage.py.""" import dis import sys @@ -33,7 +36,7 @@ class PyTracer(object): def __init__(self): # Attributes set from the collector: self.data = None - self.arcs = False + self.trace_arcs = False self.should_trace = None self.should_trace_cache = None self.warn = None @@ -65,7 +68,7 @@ class PyTracer(object): if self.last_exc_back: if frame == self.last_exc_back: # Someone forgot a return event. - if self.arcs and self.cur_file_dict: + if self.trace_arcs and self.cur_file_dict: pair = (self.last_line, -self.last_exc_firstlineno) self.cur_file_dict[pair] = None self.cur_file_dict, self.last_line = self.data_stack.pop() @@ -96,13 +99,13 @@ class PyTracer(object): # Record an executed line. if self.cur_file_dict is not None: lineno = frame.f_lineno - if self.arcs: + if self.trace_arcs: self.cur_file_dict[(self.last_line, lineno)] = None else: self.cur_file_dict[lineno] = None self.last_line = lineno elif event == 'return': - if self.arcs and self.cur_file_dict: + if self.trace_arcs and self.cur_file_dict: # Record an arc leaving the function, but beware that a # "return" event might just mean yielding from a generator. bytecode = frame.f_code.co_code[frame.f_lasti] @@ -125,6 +128,7 @@ class PyTracer(object): if self.threading: self.thread = self.threading.currentThread() sys.settrace(self._trace) + self.stopped = False return self._trace def stop(self): diff --git a/coverage/report.py b/coverage/report.py index 33a46070..df34e43f 100644 --- a/coverage/report.py +++ b/coverage/report.py @@ -1,9 +1,14 @@ -"""Reporter foundation for Coverage.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Reporter foundation for coverage.py.""" import os from coverage.files import prep_patterns, FnmatchMatcher -from coverage.misc import CoverageException, NoSource, NotPython +from coverage.misc import CoverageException, NoSource, NotPython, isolate_module + +os = isolate_module(os) class Reporter(object): @@ -29,30 +34,20 @@ class Reporter(object): def find_file_reporters(self, morfs): """Find the FileReporters we'll report on. - `morfs` is a list of modules or filenames. + `morfs` is a list of modules or file names. """ - self.file_reporters = self.coverage._get_file_reporters(morfs) + reporters = self.coverage._get_file_reporters(morfs) if self.config.include: - patterns = prep_patterns(self.config.include) - matcher = FnmatchMatcher(patterns) - filtered = [] - for fr in self.file_reporters: - if matcher.match(fr.filename): - filtered.append(fr) - self.file_reporters = filtered + matcher = FnmatchMatcher(prep_patterns(self.config.include)) + reporters = [fr for fr in reporters if matcher.match(fr.filename)] if self.config.omit: - patterns = prep_patterns(self.config.omit) - matcher = FnmatchMatcher(patterns) - filtered = [] - for fr in self.file_reporters: - if not matcher.match(fr.filename): - filtered.append(fr) - self.file_reporters = filtered + matcher = FnmatchMatcher(prep_patterns(self.config.omit)) + reporters = [fr for fr in reporters if not matcher.match(fr.filename)] - self.file_reporters.sort() + self.file_reporters = sorted(reporters) def report_files(self, report_fn, morfs, directory=None): """Run a reporting function on a number of morfs. @@ -84,5 +79,7 @@ class Reporter(object): except NotPython: # Only report errors for .py files, and only if we didn't # explicitly suppress those errors. + # NotPython is only raised by PythonFileReporter, which has a + # should_be_python() method. if fr.should_be_python() and not self.config.ignore_errors: raise diff --git a/coverage/results.py b/coverage/results.py index 7b621c18..9627373d 100644 --- a/coverage/results.py +++ b/coverage/results.py @@ -1,3 +1,6 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Results of coverage measurement.""" import collections @@ -9,19 +12,21 @@ from coverage.misc import format_lines class Analysis(object): """The results of analyzing a FileReporter.""" - def __init__(self, cov, file_reporters): - self.coverage = cov - self.file_reporter = file_reporters + def __init__(self, data, file_reporter): + self.data = data + self.file_reporter = file_reporter self.filename = self.file_reporter.filename - self.statements = self.file_reporter.statements() - self.excluded = self.file_reporter.excluded_statements() + self.statements = self.file_reporter.lines() + self.excluded = self.file_reporter.excluded_lines() # Identify missing statements. - executed = self.coverage.data.executed_lines(self.filename) + executed = self.data.lines(self.filename) or [] executed = self.file_reporter.translate_lines(executed) self.missing = self.statements - executed - if self.coverage.data.has_arcs(): + if self.data.has_arcs(): + self._arc_possibilities = sorted(self.file_reporter.arcs()) + self.exit_counts = self.file_reporter.exit_counts() self.no_branch = self.file_reporter.no_branch_lines() n_branches = self.total_branches() mba = self.missing_branch_arcs() @@ -30,8 +35,10 @@ class Analysis(object): ) n_missing_branches = sum(len(v) for k,v in iitems(mba)) else: - n_branches = n_partial_branches = n_missing_branches = 0 + self._arc_possibilities = [] + self.exit_counts = {} self.no_branch = set() + n_branches = n_partial_branches = n_missing_branches = 0 self.numbers = Numbers( n_files=1, @@ -53,15 +60,15 @@ class Analysis(object): def has_arcs(self): """Were arcs measured in this result?""" - return self.coverage.data.has_arcs() + return self.data.has_arcs() def arc_possibilities(self): """Returns a sorted list of the arcs in the code.""" - return self.file_reporter.arcs() + return self._arc_possibilities def arcs_executed(self): """Returns a sorted list of the arcs actually executed in the code.""" - executed = self.coverage.data.executed_arcs(self.filename) + executed = self.data.arcs(self.filename) or [] executed = self.file_reporter.translate_arcs(executed) return sorted(executed) @@ -113,13 +120,11 @@ class Analysis(object): def branch_lines(self): """Returns a list of line numbers that have more than one exit.""" - exit_counts = self.file_reporter.exit_counts() - return [l1 for l1,count in iitems(exit_counts) if count > 1] + return [l1 for l1,count in iitems(self.exit_counts) if count > 1] def total_branches(self): """How many total branches are there?""" - exit_counts = self.file_reporter.exit_counts() - return sum(count for count in exit_counts.values() if count > 1) + return sum(count for count in self.exit_counts.values() if count > 1) def missing_branch_arcs(self): """Return arcs that weren't executed from branch lines. @@ -142,11 +147,10 @@ class Analysis(object): (total_exits, taken_exits). """ - exit_counts = self.file_reporter.exit_counts() missing_arcs = self.missing_branch_arcs() stats = {} for lnum in self.branch_lines(): - exits = exit_counts[lnum] + exits = self.exit_counts[lnum] try: missing = len(missing_arcs[lnum]) except KeyError: diff --git a/coverage/summary.py b/coverage/summary.py index 5b8c903f..5ddbb380 100644 --- a/coverage/summary.py +++ b/coverage/summary.py @@ -1,10 +1,14 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Summary reporting""" import sys +from coverage import env from coverage.report import Reporter from coverage.results import Numbers -from coverage.misc import NotPython +from coverage.misc import NotPython, CoverageException, output_encoding class SummaryReporter(Reporter): @@ -17,55 +21,63 @@ class SummaryReporter(Reporter): def report(self, morfs, outfile=None): """Writes a report summarizing coverage statistics per module. - `outfile` is a file object to write the summary to. + `outfile` is a file object to write the summary to. It must be opened + for native strings (bytes on Python 2, Unicode on Python 3). """ self.find_file_reporters(morfs) # Prepare the formatting strings - max_name = max([len(fr.name) for fr in self.file_reporters] + [5]) - fmt_name = "%%- %ds " % max_name - fmt_err = "%s %s: %s\n" - header = (fmt_name % "Name") + " Stmts Miss" - fmt_coverage = fmt_name + "%6d %6d" + max_name = max([len(fr.relative_filename()) for fr in self.file_reporters] + [5]) + fmt_name = u"%%- %ds " % max_name + fmt_err = u"%s %s: %s\n" + fmt_skip_covered = u"\n%s file%s skipped due to complete coverage.\n" + + header = (fmt_name % "Name") + u" Stmts Miss" + fmt_coverage = fmt_name + u"%6d %6d" if self.branches: - header += " Branch BrPart" - fmt_coverage += " %6d %6d" + header += u" Branch BrPart" + fmt_coverage += u" %6d %6d" width100 = Numbers.pc_str_width() - header += "%*s" % (width100+4, "Cover") - fmt_coverage += "%%%ds%%%%" % (width100+3,) + header += u"%*s" % (width100+4, "Cover") + fmt_coverage += u"%%%ds%%%%" % (width100+3,) if self.config.show_missing: - header += " Missing" - fmt_coverage += " %s" - rule = "-" * len(header) + "\n" - header += "\n" - fmt_coverage += "\n" + header += u" Missing" + fmt_coverage += u" %s" + rule = u"-" * len(header) + u"\n" + header += u"\n" + fmt_coverage += u"\n" - if not outfile: + if outfile is None: outfile = sys.stdout + if env.PY2: + writeout = lambda u: outfile.write(u.encode(output_encoding())) + else: + writeout = outfile.write + # Write the header - outfile.write(header) - outfile.write(rule) + writeout(header) + writeout(rule) total = Numbers() + skipped_count = 0 for fr in self.file_reporters: try: analysis = self.coverage._analyze(fr) nums = analysis.numbers + total += nums if self.config.skip_covered: # Don't report on 100% files. no_missing_lines = (nums.n_missing == 0) - if self.branches: - no_missing_branches = (nums.n_partial_branches == 0) - else: - no_missing_branches = True + no_missing_branches = (nums.n_partial_branches == 0) if no_missing_lines and no_missing_branches: + skipped_count += 1 continue - args = (fr.name, nums.n_statements, nums.n_missing) + args = (fr.relative_filename(), nums.n_statements, nums.n_missing) if self.branches: args += (nums.n_branches, nums.n_partial_branches) args += (nums.pc_covered_str,) @@ -78,25 +90,32 @@ class SummaryReporter(Reporter): missing_fmtd += ", " missing_fmtd += branches_fmtd args += (missing_fmtd,) - outfile.write(fmt_coverage % args) - total += nums + writeout(fmt_coverage % args) except Exception: report_it = not self.config.ignore_errors if report_it: typ, msg = sys.exc_info()[:2] + # NotPython is only raised by PythonFileReporter, which has a + # should_be_python() method. if typ is NotPython and not fr.should_be_python(): report_it = False if report_it: - outfile.write(fmt_err % (fr.name, typ.__name__, msg)) + writeout(fmt_err % (fr.relative_filename(), typ.__name__, msg)) if total.n_files > 1: - outfile.write(rule) + writeout(rule) args = ("TOTAL", total.n_statements, total.n_missing) if self.branches: args += (total.n_branches, total.n_partial_branches) args += (total.pc_covered_str,) if self.config.show_missing: args += ("",) - outfile.write(fmt_coverage % args) + writeout(fmt_coverage % args) + + if not total.n_files and not skipped_count: + raise CoverageException("No data to report.") + + if self.config.skip_covered and skipped_count: + writeout(fmt_skip_covered % (skipped_count, 's' if skipped_count > 1 else '')) - return total.pc_covered + return total.n_statements and total.pc_covered diff --git a/coverage/templite.py b/coverage/templite.py index 9f882cf2..f131f748 100644 --- a/coverage/templite.py +++ b/coverage/templite.py @@ -1,4 +1,12 @@ -"""A simple Python template renderer, for a nano-subset of Django syntax.""" +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""A simple Python template renderer, for a nano-subset of Django syntax. + +For a detailed discussion of this code, see this chapter from 500 Lines: +http://aosabook.org/en/500L/a-template-engine.html + +""" # Coincidentally named the same as http://code.activestate.com/recipes/496702/ diff --git a/coverage/test_helpers.py b/coverage/test_helpers.py index 278b0a8a..50cc3298 100644 --- a/coverage/test_helpers.py +++ b/coverage/test_helpers.py @@ -1,7 +1,11 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """Mixin classes to help make good tests.""" import atexit import collections +import contextlib import os import random import shutil @@ -39,6 +43,56 @@ class Tee(object): return getattr(self._files[0], name) +@contextlib.contextmanager +def change_dir(new_dir): + """Change directory, and then change back. + + Use as a context manager, it will give you the new directory, and later + restore the old one. + + """ + old_dir = os.getcwd() + os.chdir(new_dir) + try: + yield os.getcwd() + finally: + os.chdir(old_dir) + + +@contextlib.contextmanager +def saved_sys_path(): + """Save sys.path, and restore it later.""" + old_syspath = sys.path[:] + try: + yield + finally: + sys.path = old_syspath + + +def setup_with_context_manager(testcase, cm): + """Use a contextmanager to setUp a test case. + + If you have a context manager you like:: + + with ctxmgr(a, b, c) as v: + # do something with v + + and you want to have that effect for a test case, call this function from + your setUp, and it will start the context manager for your test, and end it + when the test is done:: + + def setUp(self): + self.v = setup_with_context_manager(self, ctxmgr(a, b, c)) + + def test_foo(self): + # do something with self.v + + """ + val = cm.__enter__() + testcase.addCleanup(cm.__exit__, None, None, None) + return val + + class ModuleAwareMixin(TestCase): """A test case mixin that isolates changes to sys.modules.""" @@ -46,7 +100,7 @@ class ModuleAwareMixin(TestCase): super(ModuleAwareMixin, self).setUp() # Record sys.modules here so we can restore it in cleanup_modules. - self.old_modules = dict(sys.modules) + self.old_modules = list(sys.modules) self.addCleanup(self.cleanup_modules) def cleanup_modules(self): @@ -64,13 +118,7 @@ class SysPathAwareMixin(TestCase): def setUp(self): super(SysPathAwareMixin, self).setUp() - - self.old_syspath = sys.path[:] - self.addCleanup(self.cleanup_syspath) - - def cleanup_syspath(self): - """Restore the original sys.path.""" - sys.path = self.old_syspath + setup_with_context_manager(self, saved_sys_path()) class EnvironmentAwareMixin(TestCase): @@ -142,7 +190,7 @@ class TempDirMixin(SysPathAwareMixin, ModuleAwareMixin, TestCase): """A test case mixin that creates a temp directory and files in it. Includes SysPathAwareMixin and ModuleAwareMixin, because making and using - temp dirs like this will also need that kind of isolation. + temp directories like this will also need that kind of isolation. """ @@ -235,7 +283,7 @@ class TempDirMixin(SysPathAwareMixin, ModuleAwareMixin, TestCase): # We run some tests in temporary directories, because they may need to make # files for the tests. But this is expensive, so we can change per-class - # whether a temp dir is used or not. It's easy to forget to set that + # whether a temp directory is used or not. It's easy to forget to set that # option properly, so we track information about what the tests did, and # then report at the end of the process on test classes that were set # wrong. diff --git a/coverage/version.py b/coverage/version.py index 51e1310f..dc4c57c9 100644 --- a/coverage/version.py +++ b/coverage/version.py @@ -1,9 +1,33 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """The version and URL for coverage.py""" # This file is exec'ed in setup.py, don't import anything! -__version__ = "4.0a6" # see detailed history in CHANGES.txt +# Same semantics as sys.version_info. +version_info = (4, 1, 0, 'alpha', 0) + + +def _make_version(major, minor, micro, releaselevel, serial): + """Create a readable version string from version_info tuple components.""" + assert releaselevel in ['alpha', 'beta', 'candidate', 'final'] + version = "%d.%d" % (major, minor) + if micro: + version += ".%d" % (micro,) + if releaselevel != 'final': + short = {'alpha': 'a', 'beta': 'b', 'candidate': 'rc'}[releaselevel] + version += "%s%d" % (short, serial) + return version + + +def _make_url(major, minor, micro, releaselevel, serial): + """Make the URL people should start at for this version of coverage.py.""" + url = "https://coverage.readthedocs.org" + if releaselevel != 'final': + # For pre-releases, use a version-specific URL. + url += "/en/coverage-" + _make_version(major, minor, micro, releaselevel, serial) + return url + -__url__ = "https://coverage.readthedocs.org" -if max(__version__).isalpha(): - # For pre-releases, use a version-specific URL. - __url__ += "/en/" + __version__ +__version__ = _make_version(*version_info) +__url__ = _make_url(*version_info) diff --git a/coverage/xmlreport.py b/coverage/xmlreport.py index 996f19a2..50a46841 100644 --- a/coverage/xmlreport.py +++ b/coverage/xmlreport.py @@ -1,13 +1,28 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + """XML reporting for coverage.py""" import os +import os.path import sys import time import xml.dom.minidom -from coverage import __url__, __version__ +from coverage import env +from coverage import __url__, __version__, files +from coverage.misc import isolate_module from coverage.report import Reporter +os = isolate_module(os) + + +DTD_URL = ( + 'https://raw.githubusercontent.com/cobertura/web/' + 'f0366e5e2cf18f111cbd61fc34ef720a6584ba02' + '/htdocs/xml/coverage-03.dtd' +) + def rate(hit, num): """Return the fraction of `hit`/`num`, as a string.""" @@ -20,19 +35,22 @@ def rate(hit, num): class XmlReporter(Reporter): """A reporter for writing Cobertura-style XML coverage results.""" - def __init__(self, coverage, config, file_locator): + def __init__(self, coverage, config): super(XmlReporter, self).__init__(coverage, config) - self.file_locator = file_locator self.source_paths = set() + if config.source: + for src in config.source: + if os.path.exists(src): + self.source_paths.add(files.canonical_filename(src)) self.packages = {} self.xml_out = None - self.arcs = coverage.data.has_arcs() + self.has_arcs = coverage.data.has_arcs() def report(self, morfs, outfile=None): """Generate a Cobertura-compatible XML report for `morfs`. - `morfs` is a list of modules or filenames. + `morfs` is a list of modules or file names. `outfile` is a file object to write the XML to. @@ -42,11 +60,7 @@ class XmlReporter(Reporter): # Create the DOM that will store the data. impl = xml.dom.minidom.getDOMImplementation() - docType = impl.createDocumentType( - "coverage", None, - "http://cobertura.sourceforge.net/xml/coverage-03.dtd" - ) - self.xml_out = impl.createDocument(None, "coverage", docType) + self.xml_out = impl.createDocument(None, "coverage", None) # Write header stuff. xcoverage = self.xml_out.documentElement @@ -55,6 +69,7 @@ class XmlReporter(Reporter): xcoverage.appendChild(self.xml_out.createComment( " Generated by coverage.py: %s " % __url__ )) + xcoverage.appendChild(self.xml_out.createComment(" Based on %s " % DTD_URL)) # Call xml_file for each file in the data. self.report_files(self.xml_file, morfs) @@ -87,7 +102,7 @@ class XmlReporter(Reporter): xclasses.appendChild(class_elts[class_name]) xpackage.setAttribute("name", pkg_name.replace(os.sep, '.')) xpackage.setAttribute("line-rate", rate(lhits, lnum)) - if self.arcs: + if self.has_arcs: branch_rate = rate(bhits, bnum) else: branch_rate = "0" @@ -100,14 +115,17 @@ class XmlReporter(Reporter): bhits_tot += bhits xcoverage.setAttribute("line-rate", rate(lhits_tot, lnum_tot)) - if self.arcs: + if self.has_arcs: branch_rate = rate(bhits_tot, bnum_tot) else: branch_rate = "0" xcoverage.setAttribute("branch-rate", branch_rate) # Use the DOM to write the output file. - outfile.write(self.xml_out.toprettyxml()) + out = self.xml_out.toprettyxml() + if env.PY2: + out = out.encode("utf8") + outfile.write(out) # Return the total percentage. denom = lnum_tot + bnum_tot @@ -122,15 +140,16 @@ class XmlReporter(Reporter): # Create the 'lines' and 'package' XML elements, which # are populated later. Note that a package == a directory. - filename = self.file_locator.relative_filename(fr.filename) + filename = fr.relative_filename() filename = filename.replace("\\", "/") dirname = os.path.dirname(filename) or "." parts = dirname.split("/") dirname = "/".join(parts[:self.config.xml_package_depth]) package_name = dirname.replace("/", ".") - className = fr.name + rel_name = fr.relative_filename() - self.source_paths.add(self.file_locator.relative_dir.rstrip('/')) + if rel_name != fr.filename: + self.source_paths.add(fr.filename[:-len(rel_name)].rstrip(r"\/")) package = self.packages.setdefault(package_name, [{}, 0, 0, 0, 0]) xclass = self.xml_out.createElement("class") @@ -145,6 +164,7 @@ class XmlReporter(Reporter): xclass.setAttribute("complexity", "0") branch_stats = analysis.branch_stats() + missing_branch_arcs = analysis.missing_branch_arcs() # For each statement, create an XML 'line' element. for line in sorted(analysis.statements): @@ -155,7 +175,7 @@ class XmlReporter(Reporter): # executed? If so, that should be recorded here. xline.setAttribute("hits", str(int(line not in analysis.missing))) - if self.arcs: + if self.has_arcs: if line in branch_stats: total, taken = branch_stats[line] xline.setAttribute("branch", "true") @@ -163,12 +183,15 @@ class XmlReporter(Reporter): "condition-coverage", "%d%% (%d/%d)" % (100*taken/total, taken, total) ) + if line in missing_branch_arcs: + annlines = ["exit" if b < 0 else str(b) for b in missing_branch_arcs[line]] + xline.setAttribute("missing-branches", ",".join(annlines)) xlines.appendChild(xline) class_lines = len(analysis.statements) class_hits = class_lines - len(analysis.missing) - if self.arcs: + if self.has_arcs: class_branches = sum(t for t, k in branch_stats.values()) missing_branches = sum(t - k for t, k in branch_stats.values()) class_br_hits = class_branches - missing_branches @@ -178,13 +201,13 @@ class XmlReporter(Reporter): # Finalize the statistics that are collected in the XML DOM. xclass.setAttribute("line-rate", rate(class_hits, class_lines)) - if self.arcs: + if self.has_arcs: branch_rate = rate(class_br_hits, class_branches) else: branch_rate = "0" xclass.setAttribute("branch-rate", branch_rate) - package[0][className] = xclass + package[0][rel_name] = xclass package[1] += class_hits package[2] += class_lines package[3] += class_br_hits |