make --source and -m play nice together

--HG--
branch : __main__-support
extra : rebase_source : c9ca9fddecafddd5ef4db9cc64ca1c0972130aab

7 files changed, 221 insertions, 55 deletions

diff --git a/coverage/cmdline.py b/coverage/cmdline.py
index e7efe5c4..a80d1168 100644
--- a/coverage/cmdline.py
+++ b/coverage/cmdline.py
@@ -519,9 +519,10 @@ class CoverageScript(object):
 
         # Set the first path element properly.
         old_path0 = sys.path[0]
+        main_module = args[0] if options.module else None
 
         # Run the script.
-        self.coverage.start()
+        self.coverage.start(main_module=main_module)
         code_ran = True
         try:
             if options.module:
diff --git a/coverage/control.py b/coverage/control.py
index 1191b9eb..5e154537 100644
--- a/coverage/control.py
+++ b/coverage/control.py
@@ -1,6 +1,6 @@
 """Core control stuff for Coverage."""
 
-import atexit, os, platform, random, socket, sys
+import atexit, inspect, os, platform, random, socket, sys
 
 from coverage.annotate import AnnotateReporter
 from coverage.backward import string_class, iitems
@@ -12,6 +12,7 @@ from coverage.debug import DebugControl
 from coverage.plugin import CoveragePlugin, Plugins, overrides
 from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher
 from coverage.files import PathAliases, find_python_files, prep_patterns
+from coverage.files import ModuleMatcher
 from coverage.html import HtmlReporter
 from coverage.misc import CoverageException, bool_or_none, join_regex
 from coverage.misc import file_be_gone
@@ -140,7 +141,7 @@ class Coverage(object):
 
         # Other instance attributes, set later.
         self.omit = self.include = self.source = None
-        self.source_pkgs = self.file_locator = None
+        self.not_imported = self.source_pkgs = self.file_locator = None
         self.data = self.collector = None
         self.plugins = self.file_tracers = None
         self.pylib_dirs = self.cover_dir = None
@@ -198,6 +199,7 @@ class Coverage(object):
                 self.source.append(self.file_locator.canonical_filename(src))
             else:
                 self.source_pkgs.append(src)
+        self.not_imported = list(self.source_pkgs)
 
         self.omit = prep_patterns(self.config.omit)
         self.include = prep_patterns(self.config.include)
@@ -269,6 +271,7 @@ class Coverage(object):
         # Create the matchers we need for _should_trace
         if self.source or self.source_pkgs:
             self.source_match = TreeMatcher(self.source)
+            self.source_pkgs_match = ModuleMatcher(self.source_pkgs)
         else:
             if self.cover_dir:
                 self.cover_match = TreeMatcher([self.cover_dir])
@@ -303,6 +306,34 @@ class Coverage(object):
                 filename = filename[:-9] + ".py"
         return filename
 
+    def _name_for_module(self, module_namespace, filename):
+        """
+        For configurability's sake, we allow __main__ modules to be matched by their importable name.
+
+        If loaded via runpy (aka -m), we can usually recover the "original" full dotted module name,
+        otherwise, we resort to interpreting the filename to get the module's name.
+        In the case that the module name can't be deteremined, None is returned.
+        """
+        # TODO: unit-test
+        dunder_name = module_namespace.get('__name__', None)
+
+        if isinstance(dunder_name, str) and dunder_name != '__main__':
+            # this is the usual case: an imported module
+            return dunder_name
+
+        loader = module_namespace.get('__loader__', None)
+        if hasattr(loader, 'fullname') and isinstance(loader.fullname, str):
+            # module loaded via runpy -m
+            return loader.fullname
+
+        # script as first argument to python cli
+        inspectedname = inspect.getmodulename(filename)
+        if inspectedname is not None:
+            return inspectedname
+        else:
+            return dunder_name
+
+
     def _should_trace_with_reason(self, filename, frame):
         """Decide whether to trace execution in `filename`, with a reason.
 
@@ -318,8 +349,6 @@ class Coverage(object):
             disp.reason = reason
             return disp
 
-        self._check_for_packages()
-
         # Compiled Python files have two filenames: frame.f_code.co_filename is
         # the filename at the time the .pyc was compiled.  The second name is
         # __file__, which is where the .pyc was actually loaded from.  Since
@@ -334,6 +363,8 @@ class Coverage(object):
             # Empty string is pretty useless
             return nope(disp, "empty string isn't a filename")
 
+        modulename = self._name_for_module(frame.f_globals, filename)
+
         if filename.startswith('memory:'):
             return nope(disp, "memory isn't traceable")
 
@@ -373,7 +404,9 @@ class Coverage(object):
                         (plugin, disp.original_filename)
                     )
                 if disp.check_filters:
-                    reason = self._check_include_omit_etc(disp.source_filename)
+                    reason = self._check_include_omit_etc(
+                            disp.source_filename, modulename,
+                    )
                     if reason:
                         nope(disp, reason)
 
@@ -381,7 +414,7 @@ class Coverage(object):
 
         return nope(disp, "no plugin found")  # TODO: a test that causes this.
 
-    def _check_include_omit_etc(self, filename):
+    def _check_include_omit_etc(self, filename, modulename):
         """Check a filename against the include, omit, etc, rules.
 
         Returns a string or None.  String means, don't trace, and is the reason
@@ -393,6 +426,12 @@ class Coverage(object):
         # any canned exclusions. If they didn't, then we have to exclude the
         # stdlib and coverage.py directories.
         if self.source_match:
+            match = self.source_pkgs_match.match(modulename)
+            if match:
+                if modulename in self.not_imported:
+                    self.not_imported.remove(modulename)
+                return None  # There's no reason to skip this file.
+
             if not self.source_match.match(filename):
                 return "falls outside the --source trees"
         elif self.include_match:
@@ -448,46 +487,6 @@ class Coverage(object):
         self._warnings.append(msg)
         sys.stderr.write("Coverage.py warning: %s\n" % msg)
 
-    def _check_for_packages(self):
-        """Update the source_match matcher with latest imported packages."""
-        # Our self.source_pkgs attribute is a list of package names we want to
-        # measure.  Each time through here, we see if we've imported any of
-        # them yet.  If so, we add its file to source_match, and we don't have
-        # to look for that package any more.
-        if self.source_pkgs:
-            found = []
-            for pkg in self.source_pkgs:
-                try:
-                    mod = sys.modules[pkg]
-                except KeyError:
-                    continue
-
-                found.append(pkg)
-
-                try:
-                    pkg_file = mod.__file__
-                except AttributeError:
-                    pkg_file = None
-                else:
-                    d, f = os.path.split(pkg_file)
-                    if f.startswith('__init__'):
-                        # This is actually a package, return the directory.
-                        pkg_file = d
-                    else:
-                        pkg_file = self._source_for_file(pkg_file)
-                    pkg_file = self.file_locator.canonical_filename(pkg_file)
-                    if not os.path.exists(pkg_file):
-                        pkg_file = None
-
-                if pkg_file:
-                    self.source.append(pkg_file)
-                    self.source_match.add(pkg_file)
-                else:
-                    self._warn("Module %s has no Python source." % pkg)
-
-            for pkg in found:
-                self.source_pkgs.remove(pkg)
-
     def use_cache(self, usecache):
         """Control the use of a data file (incorrectly called a cache).
 
@@ -652,11 +651,21 @@ class Coverage(object):
         self.data.add_plugin_data(self.collector.get_plugin_data())
         self.collector.reset()
 
-        # If there are still entries in the source_pkgs list, then we never
+        # If there are still entries in the not_imported list, then we never
         # encountered those packages.
         if self._warn_unimported_source:
-            for pkg in self.source_pkgs:
-                self._warn("Module %s was never imported." % pkg)
+            for pkg in self.not_imported:
+                if pkg not in sys.modules:
+                    self._warn("Module %s was never imported." % pkg)
+                elif not hasattr(sys.modules[pkg], '__file__'):
+                    self._warn("Module %s has no Python source." % pkg)
+                else:
+                    raise AssertionError('''\
+Unexpected third case:
+    name: %s
+    object: %r
+    __file__: %s''' % (pkg, sys.modules[pkg], sys.modules[pkg].__file__)
+                    )
 
         # Find out if we got any data.
         summary = self.data.summary()
diff --git a/coverage/files.py b/coverage/files.py
index 1ed7276e..5e7c35aa 100644
--- a/coverage/files.py
+++ b/coverage/files.py
@@ -173,6 +173,50 @@ class TreeMatcher(object):
         return False
 
 
+class ModuleMatcher(object):
+    """A matcher for files in a tree."""
+    def __init__(self, module_names, main_module=None):
+        self.modules = list(module_names)
+        self.main_module = main_module
+        self.not_imported = list(module_names)
+
+    def __repr__(self):
+        if self.main_module:
+            main_module = ', main_module=%r' % self.main_module
+        else:
+            main_module = ''
+        return "<ModuleMatcher %r%s>" % (self.modules, main_module)
+
+    def info(self):
+        """A list of strings for displaying when dumping state."""
+        return ['main_module=%r' % self.main_module] + self.modules
+
+    def add(self, module):
+        """Add another directory to the list we match for."""
+        self.modules.append(module)
+
+    def match(self, module_name):
+        """Does `module_name` indicate a module in one of our packages?
+
+        On success, returns the matched module name, which can be different in
+        the case of __main__.
+        """
+        if not module_name:
+            return False
+        elif module_name == '__main__':
+            module_name = self.main_module or module_name
+
+        for m in self.modules:
+            if module_name.startswith(m):
+                if module_name == m:
+                    return module_name
+                if module_name[len(m)] == '.':
+                    # This is a module in the package
+                    return module_name
+
+        return False
+
+
 class FnmatchMatcher(object):
     """A matcher for files by filename pattern."""
     def __init__(self, pats):
diff --git a/doc/contributing.rst b/doc/contributing.rst
index 88c99a99..8e1fd723 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -129,7 +129,7 @@ I try to keep the coverage.py as clean as possible.  I use pylint to alert me
 to possible problems::
 
     $ make lint
-    pylint --rcfile=.pylintrc coverage setup.py tests
+    pylint coverage setup.py tests
     python -m tabnanny coverage setup.py tests
     python igor.py check_eol
 
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 695c3bec..3032a2f1 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -33,6 +33,9 @@ class BaseCmdLineTest(CoverageTest):
         ignore_errors=None, include=None, omit=None, morfs=[],
         show_missing=None,
     )
+    defaults.start(
+        main_module=None,
+    )
     defaults.xml_report(
         ignore_errors=None, include=None, omit=None, morfs=[], outfile=None,
     )
@@ -472,7 +475,7 @@ class CmdLineTest(BaseCmdLineTest):
         self.cmd_executes("run -m mymodule", """\
             .coverage()
             .erase()
-            .start()
+            .start(main_module='mymodule')
             .run_python_module('mymodule', ['mymodule'])
             .stop()
             .save()
@@ -480,7 +483,7 @@ class CmdLineTest(BaseCmdLineTest):
         self.cmd_executes("run -m mymodule -qq arg1 arg2", """\
             .coverage()
             .erase()
-            .start()
+            .start(main_module='mymodule')
             .run_python_module('mymodule', ['mymodule', '-qq', 'arg1', 'arg2'])
             .stop()
             .save()
@@ -488,7 +491,7 @@ class CmdLineTest(BaseCmdLineTest):
         self.cmd_executes("run --branch -m mymodule", """\
             .coverage(branch=True)
             .erase()
-            .start()
+            .start(main_module='mymodule')
             .run_python_module('mymodule', ['mymodule'])
             .stop()
             .save()
diff --git a/tests/test_files.py b/tests/test_files.py
index 648c76a9..d72788c4 100644
--- a/tests/test_files.py
+++ b/tests/test_files.py
@@ -2,7 +2,9 @@
 
 import os, os.path
 
-from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher
+from coverage.files import (
+    FileLocator, TreeMatcher, FnmatchMatcher, ModuleMatcher
+)
 from coverage.files import PathAliases, find_python_files, abs_file
 from coverage.misc import CoverageException
 
@@ -80,6 +82,64 @@ class MatcherTest(CoverageTest):
         for filepath, matches in matches_to_try:
             self.assertMatches(tm, filepath, matches)
 
+    def test_module_matcher(self):
+        matches_to_try = [
+            ('test', True),
+            ('test', True),
+            ('trash', False),
+            ('testing', False),
+            ('test.x', True),
+            ('test.x.y.z', True),
+            ('py', False),
+            ('py.t', False),
+            ('py.test', True),
+            ('py.testing', False),
+            ('py.test.buz', True),
+            ('py.test.buz.baz', True),
+            ('__main__', False),
+            ('mymain', True),
+            ('yourmain', False),
+        ]
+        modules = ['test', 'py.test', 'mymain']
+        for mm in (
+            ModuleMatcher(modules),
+            ModuleMatcher(modules, main_module=None),
+            ModuleMatcher(modules, main_module='yourmain'),
+        ):
+            self.assertEqual(
+                mm.info(),
+                ['main_module=%r' % mm.main_module] + modules
+            )
+            for modulename, matches in matches_to_try:
+                self.assertEqual(
+                    mm.match(modulename),
+                    modulename if matches else False,
+                    modulename,
+                )
+
+    def test_module_matcher_dunder_main(self):
+        matches_to_try = [
+            ('__main__', True),
+            ('mymain', True),
+            ('yourmain', False),
+        ]
+        modules = ['test', 'py.test', 'mymain']
+        mm = ModuleMatcher(modules, main_module='mymain')
+        self.assertEqual(mm.info(), ["main_module='mymain'"] + modules)
+        for modulename, matches in matches_to_try:
+            if not matches:
+                expected = False
+            elif modulename == '__main__':
+                expected = mm.main_module
+            else:
+                expected = modulename
+
+            self.assertEqual(
+                mm.match(modulename),
+                expected,
+                modulename,
+            )
+
     def test_fnmatch_matcher(self):
         matches_to_try = [
             (self.make_file("sub/file1.py"), True),
diff --git a/tests/test_process.py b/tests/test_process.py
index ac5c6e1b..e53e64c9 100644
--- a/tests/test_process.py
+++ b/tests/test_process.py
@@ -331,6 +331,55 @@ class ProcessTest(CoverageTest):
         out_py = self.run_command("python -m tests.try_execfile")
         self.assertMultiLineEqual(out_cov, out_py)
 
+    def test_coverage_run_dashm_equal_to_doubledashsource(self):
+        """regression test for #328
+
+        When imported by -m, a module's __name__ is __main__, but we need the
+        --source machinery to know and respect the original name.
+        """
+        # These -m commands assume the coverage tree is on the path.
+        out_cov = self.run_command(
+            "coverage run --source tests.try_execfile -m tests.try_execfile"
+        )
+        out_py = self.run_command("python -m tests.try_execfile")
+        self.assertMultiLineEqual(out_cov, out_py)
+
+    def test_coverage_run_dashm_superset_of_doubledashsource(self):
+        """Edge case: --source foo -m foo.bar"""
+        # These -m commands assume the coverage tree is on the path.
+        out_cov = self.run_command(
+            "coverage run --source tests -m tests.try_execfile"
+        )
+        out_py = self.run_command("python -m tests.try_execfile")
+        self.assertMultiLineEqual(out_cov, out_py)
+
+        st, out = self.run_command_status("coverage report")
+        self.assertEqual(st, 0)
+        self.assertEqual(self.line_count(out), 6, out)
+
+    def test_coverage_run_script_imports_doubledashsource(self):
+        self.make_file("myscript", """\
+            import sys
+            sys.dont_write_bytecode = True
+
+            def main():
+                import tests.try_execfile
+
+            if __name__ == '__main__':
+                main()
+            """)
+
+        # These -m commands assume the coverage tree is on the path.
+        out_cov = self.run_command(
+            "coverage run --source tests myscript"
+        )
+        out_py = self.run_command("python myscript")
+        self.assertMultiLineEqual(out_cov, out_py)
+
+        st, out = self.run_command_status("coverage report")
+        self.assertEqual(st, 0)
+        self.assertEqual(self.line_count(out), 6, out)
+
     def test_coverage_run_dashm_is_like_python_dashm_off_path(self):
         # https://bitbucket.org/ned/coveragepy/issue/242
         tryfile = os.path.join(here, "try_execfile.py")