feat: warnings are now real warnings

This makes coverage warnings visible when running test suites under
pytest.  But it also means some uninteresting warnings would show up in
our own test suite, so we had to catch or suppress those.

14 files changed, 183 insertions, 89 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 1dd5c175..205ef0ab 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -33,6 +33,8 @@ Unreleased
   imported a file that will be measured" warnings about Django itself.  These
   have been fixed, closing `issue 1150`_.
 
+- Warnings generated by coverage.py are now real Python warnings.
+
 - The ``COVERAGE_DEBUG_FILE`` environment variable now accepts ``stdout`` and
   ``stderr`` to write to those destinations.
 
diff --git a/coverage/control.py b/coverage/control.py
index bf91d447..b13acf45 100644
--- a/coverage/control.py
+++ b/coverage/control.py
@@ -11,6 +11,7 @@ import os.path
 import platform
 import sys
 import time
+import warnings
 
 from coverage import env
 from coverage.annotate import AnnotateReporter
@@ -20,7 +21,7 @@ from coverage.context import should_start_context_test_function, combine_context
 from coverage.data import CoverageData, combine_parallel_data
 from coverage.debug import DebugControl, short_stack, write_formatted_info
 from coverage.disposition import disposition_debug_msg
-from coverage.exceptions import CoverageException
+from coverage.exceptions import CoverageException, CoverageWarning
 from coverage.files import PathAliases, abs_file, relative_filename, set_relative_directory
 from coverage.html import HtmlReporter
 from coverage.inorout import InOrOut
@@ -362,7 +363,7 @@ class Coverage:
             msg = f"{msg} ({slug})"
         if self._debug.should('pid'):
             msg = f"[{os.getpid()}] {msg}"
-        sys.stderr.write(f"Coverage.py warning: {msg}\n")
+        warnings.warn(msg, category=CoverageWarning, stacklevel=2)
 
         if once:
             self._no_warn_slugs.append(slug)
diff --git a/coverage/exceptions.py b/coverage/exceptions.py
index ed96fb21..6631e1ad 100644
--- a/coverage/exceptions.py
+++ b/coverage/exceptions.py
@@ -46,3 +46,8 @@ class StopEverything(BaseCoverageException):
 
     """
     pass
+
+
+class CoverageWarning(Warning):
+    """A warning from Coverage.py."""
+    pass
diff --git a/coverage/inorout.py b/coverage/inorout.py
index fae9ef18..32eb9079 100644
--- a/coverage/inorout.py
+++ b/coverage/inorout.py
@@ -356,10 +356,9 @@ class InOrOut:
                         )
                     break
             except Exception:
-                self.warn(
-                    "Disabling plug-in %r due to an exception:" % (plugin._coverage_plugin_name)
-                )
-                traceback.print_exc()
+                plugin_name = plugin._coverage_plugin_name
+                tb = traceback.format_exc()
+                self.warn(f"Disabling plug-in {plugin_name!r} due to an exception:\n{tb}")
                 plugin._coverage_enabled = False
                 continue
         else:
@@ -503,10 +502,8 @@ class InOrOut:
         # The module was in sys.modules, and seems like a module with code, but
         # we never measured it. I guess that means it was imported before
         # coverage even started.
-        self.warn(
-            "Module %s was previously imported, but not measured" % pkg,
-            slug="module-not-measured",
-        )
+        msg = f"Module {pkg} was previously imported, but not measured"
+        self.warn(msg, slug="module-not-measured")
 
     def find_possibly_unexecuted_files(self):
         """Find files in the areas of interest that might be untraced.
diff --git a/coverage/pytracer.py b/coverage/pytracer.py
index 51f08a1b..540df68c 100644
--- a/coverage/pytracer.py
+++ b/coverage/pytracer.py
@@ -254,10 +254,8 @@ class PyTracer:
             # has changed to None.
             dont_warn = (env.PYPY and env.PYPYVERSION >= (5, 4) and self.in_atexit and tf is None)
             if (not dont_warn) and tf != self._trace:   # pylint: disable=comparison-with-callable
-                self.warn(
-                    f"Trace function changed, measurement is likely wrong: {tf!r}",
-                    slug="trace-changed",
-                )
+                msg = f"Trace function changed, measurement is likely wrong: {tf!r}"
+                self.warn(msg, slug="trace-changed")
 
     def activity(self):
         """Has there been any activity?"""
diff --git a/tests/helpers.py b/tests/helpers.py
index 21459cd4..369875b9 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -14,6 +14,7 @@ import textwrap
 
 from unittest import mock
 
+from coverage.exceptions import CoverageWarning
 from coverage.misc import output_encoding
 
 
@@ -262,3 +263,17 @@ def assert_count_equal(a, b):
     This only works for hashable elements.
     """
     assert collections.Counter(list(a)) == collections.Counter(list(b))
+
+
+def assert_coverage_warnings(warns, *msgs):
+    """
+    Assert that `warns` are all CoverageWarning's, and have `msgs` as messages.
+    """
+    assert msgs     # don't call this without some messages.
+    assert len(warns) == len(msgs)
+    assert all(w.category == CoverageWarning for w in warns)
+    for actual, expected in zip((w.message.args[0] for w in warns), msgs):
+        if hasattr(expected, "search"):
+            assert expected.search(actual), f"{actual!r} didn't match {expected!r}"
+        else:
+            assert expected == actual
diff --git a/tests/test_api.py b/tests/test_api.py
index d6a9c08a..885f3370 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -23,7 +23,7 @@ from coverage.files import abs_file, relative_filename
 from coverage.misc import import_local_file
 
 from tests.coveragetest import CoverageTest, TESTS_DIR, UsingModulesMixin
-from tests.helpers import assert_count_equal, change_dir, nice_file
+from tests.helpers import assert_count_equal, assert_coverage_warnings, change_dir, nice_file
 
 
 class ApiTest(CoverageTest):
@@ -300,9 +300,11 @@ class ApiTest(CoverageTest):
         # If nothing was measured, the file-touching didn't happen properly.
         self.make_file("foo/bar.py", "print('Never run')")
         self.make_file("test.py", "assert True")
-        cov = coverage.Coverage(source=["foo"])
-        self.start_import_stop(cov, "test")
-        cov.report()
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage(source=["foo"])
+            self.start_import_stop(cov, "test")
+            cov.report()
+        assert_coverage_warnings(warns, "No data was collected. (no-data-collected)")
         # Name         Stmts   Miss  Cover
         # --------------------------------
         # foo/bar.py       1      1     0%
@@ -517,18 +519,19 @@ class ApiTest(CoverageTest):
             import sys, os
             print("Hello")
             """)
-        cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
-        self.start_import_stop(cov, "hello")
-        cov.get_data()
-
-        out, err = self.stdouterr()
-        assert "Hello\n" in out
-        assert textwrap.dedent("""\
-            Coverage.py warning: Module sys has no Python source. (module-not-python)
-            Coverage.py warning: Module xyzzy was never imported. (module-not-imported)
-            Coverage.py warning: Module quux was never imported. (module-not-imported)
-            Coverage.py warning: No data was collected. (no-data-collected)
-            """) in err
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
+            self.start_import_stop(cov, "hello")
+            cov.get_data()
+
+        assert "Hello\n" == self.stdout()
+        assert_coverage_warnings(
+            warns,
+            "Module sys has no Python source. (module-not-python)",
+            "Module xyzzy was never imported. (module-not-imported)",
+            "Module quux was never imported. (module-not-imported)",
+            "No data was collected. (no-data-collected)",
+            )
 
     def test_warnings_suppressed(self):
         self.make_file("hello.py", """\
@@ -539,24 +542,25 @@ class ApiTest(CoverageTest):
             [run]
             disable_warnings = no-data-collected, module-not-imported
             """)
-        cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
-        self.start_import_stop(cov, "hello")
-        cov.get_data()
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
+            self.start_import_stop(cov, "hello")
+            cov.get_data()
 
-        out, err = self.stdouterr()
-        assert "Hello\n" in out
-        assert "Coverage.py warning: Module sys has no Python source. (module-not-python)" in err
-        assert "module-not-imported" not in err
-        assert "no-data-collected" not in err
+        assert "Hello\n" == self.stdout()
+        assert_coverage_warnings(warns, "Module sys has no Python source. (module-not-python)")
+        # No "module-not-imported" in warns
+        # No "no-data-collected" in warns
 
     def test_warn_once(self):
-        cov = coverage.Coverage()
-        cov.load()
-        cov._warn("Warning, warning 1!", slug="bot", once=True)
-        cov._warn("Warning, warning 2!", slug="bot", once=True)
-        err = self.stderr()
-        assert "Warning, warning 1!" in err
-        assert "Warning, warning 2!" not in err
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage()
+            cov.load()
+            cov._warn("Warning, warning 1!", slug="bot", once=True)
+            cov._warn("Warning, warning 2!", slug="bot", once=True)
+
+        assert_coverage_warnings(warns, "Warning, warning 1! (bot)")
+        # No "Warning, warning 2!" in warns
 
     def test_source_and_include_dont_conflict(self):
         # A bad fix made this case fail: https://github.com/nedbat/coveragepy/issues/541
@@ -683,12 +687,12 @@ class ApiTest(CoverageTest):
         cov = coverage.Coverage(source=["."])
         cov.set_option("run:dynamic_context", "test_function")
         cov.start()
-        # Switch twice, but only get one warning.
-        cov.switch_context("test1")                                     # pragma: nested
-        cov.switch_context("test2")                                     # pragma: nested
-        expected = "Coverage.py warning: Conflicting dynamic contexts (dynamic-conflict)\n"
-        assert expected == self.stderr()
+        with pytest.warns(Warning) as warns:
+            # Switch twice, but only get one warning.
+            cov.switch_context("test1")                                 # pragma: nested
+            cov.switch_context("test2")                                 # pragma: nested
         cov.stop()                                                      # pragma: nested
+        assert_coverage_warnings(warns, "Conflicting dynamic contexts (dynamic-conflict)")
 
     def test_switch_context_unstarted(self):
         # Coverage must be started to switch context
diff --git a/tests/test_html.py b/tests/test_html.py
index c9dbacc8..56519a64 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -24,7 +24,7 @@ from coverage.report import get_analysis_to_report
 from tests.coveragetest import CoverageTest, TESTS_DIR
 from tests.goldtest import gold_path
 from tests.goldtest import compare, contains, doesnt_contain, contains_any
-from tests.helpers import change_dir
+from tests.helpers import assert_coverage_warnings, change_dir
 
 
 class HtmlTestHelpers(CoverageTest):
@@ -341,13 +341,14 @@ class HtmlWithUnparsableFilesTest(HtmlTestHelpers, CoverageTest):
         self.make_file("innocuous.py", "a = 2")
         cov = coverage.Coverage()
         self.start_import_stop(cov, "main")
+
         self.make_file("innocuous.py", "<h1>This isn't python!</h1>")
-        cov.html_report(ignore_errors=True)
-        msg = "Expected a warning to be thrown when an invalid python file is parsed"
-        assert 1 == len(cov._warnings), msg
-        msg = "Warning message should be in 'invalid file' warning"
-        assert "Couldn't parse Python file" in cov._warnings[0], msg
-        assert "innocuous.py" in cov._warnings[0], "Filename should be in 'invalid file' warning"
+        with pytest.warns(Warning) as warns:
+            cov.html_report(ignore_errors=True)
+        assert_coverage_warnings(
+            warns,
+            re.compile(r"Couldn't parse Python file '.*innocuous.py' \(couldnt-parse\)"),
+            )
         self.assert_exists("htmlcov/index.html")
         # This would be better as a glob, if the HTML layout changes:
         self.assert_doesnt_exist("htmlcov/innocuous.html")
diff --git a/tests/test_oddball.py b/tests/test_oddball.py
index 52f80734..a97fc190 100644
--- a/tests/test_oddball.py
+++ b/tests/test_oddball.py
@@ -81,17 +81,18 @@ class RecursionTest(CoverageTest):
     def test_long_recursion(self):
         # We can't finish a very deep recursion, but we don't crash.
         with pytest.raises(RuntimeError):
-            self.check_coverage("""\
-                def recur(n):
-                    if n == 0:
-                        return 0
-                    else:
-                        return recur(n-1)+1
-
-                recur(100000)  # This is definitely too many frames.
-                """,
-                [1, 2, 3, 5, 7], ""
-                )
+            with pytest.warns(None):
+                self.check_coverage("""\
+                    def recur(n):
+                        if n == 0:
+                            return 0
+                        else:
+                            return recur(n-1)+1
+
+                    recur(100000)  # This is definitely too many frames.
+                    """,
+                    [1, 2, 3, 5, 7], ""
+                    )
 
     def test_long_recursion_recovery(self):
         # Test the core of bug 93: https://github.com/nedbat/coveragepy/issues/93
@@ -117,7 +118,8 @@ class RecursionTest(CoverageTest):
             """)
 
         cov = coverage.Coverage()
-        self.start_import_stop(cov, "recur")
+        with pytest.warns(None):
+            self.start_import_stop(cov, "recur")
 
         pytrace = (cov._collector.tracer_name() == "PyTracer")
         expected_missing = [3]
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 3401895b..b15ee45b 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -14,7 +14,7 @@ import coverage
 from coverage import env
 from coverage.control import Plugins
 from coverage.data import line_counts
-from coverage.exceptions import CoverageException
+from coverage.exceptions import CoverageException, CoverageWarning
 from coverage.misc import import_local_file
 
 import coverage.plugin
@@ -193,7 +193,9 @@ class PluginTest(CoverageTest):
         cov = coverage.Coverage(debug=["sys"])
         cov._debug_file = debug_out
         cov.set_option("run:plugins", ["plugin_sys_info"])
-        cov.start()
+        with pytest.warns(None):
+            # Catch warnings so we don't see "plugins aren't supported on PyTracer"
+            cov.start()
         cov.stop()      # pragma: nested
 
         out_lines = [line.strip() for line in debug_out.getvalue().splitlines()]
@@ -631,28 +633,29 @@ class BadFileTracerTest(FileTracerTest):
         explaining why.
 
         """
-        self.run_plugin(module_name)
+        with pytest.warns(Warning) as warns:
+            self.run_plugin(module_name)
 
         stderr = self.stderr()
-
+        stderr += "".join(w.message.args[0] for w in warns)
         if our_error:
-            errors = stderr.count("# Oh noes!")
             # The exception we're causing should only appear once.
-            assert errors == 1
+            assert stderr.count("# Oh noes!") == 1
 
         # There should be a warning explaining what's happening, but only one.
         # The message can be in two forms:
         #   Disabling plug-in '...' due to previous exception
         # or:
         #   Disabling plug-in '...' due to an exception:
-        msg = f"Disabling plug-in '{module_name}.{plugin_name}' due to "
-        warnings = stderr.count(msg)
-        assert warnings == 1
+        assert len(warns) == 1
+        assert issubclass(warns[0].category, CoverageWarning)
+        warnmsg = warns[0].message.args[0]
+        assert f"Disabling plug-in '{module_name}.{plugin_name}' due to " in warnmsg
 
         if excmsg:
             assert excmsg in stderr
         if excmsgs:
-            assert any(em in stderr for em in excmsgs), "expected one of %r" % excmsgs
+            assert any(em in stderr for em in excmsgs), f"expected one of {excmsgs} in stderr"
 
     def test_file_tracer_has_no_file_tracer_method(self):
         self.make_file("bad_plugin.py", """\
diff --git a/tests/test_process.py b/tests/test_process.py
index a912debb..54bf345d 100644
--- a/tests/test_process.py
+++ b/tests/test_process.py
@@ -130,7 +130,7 @@ class ProcessTest(CoverageTest):
         self.assert_exists(".coverage")
         self.assert_exists(".coverage.bad")
         warning_regex = (
-            r"Coverage.py warning: Couldn't use data file '.*\.coverage\.bad': "
+            r"CoverageWarning: Couldn't use data file '.*\.coverage\.bad': "
             r"file (is encrypted or )?is not a database"
         )
         assert re.search(warning_regex, out)
@@ -163,7 +163,7 @@ class ProcessTest(CoverageTest):
         for n in "12":
             self.assert_exists(f".coverage.bad{n}")
             warning_regex = (
-                r"Coverage.py warning: Couldn't use data file '.*\.coverage.bad{}': "
+                r"CoverageWarning: Couldn't use data file '.*\.coverage.bad{}': "
                 r"file (is encrypted or )?is not a database"
                 .format(n)
             )
@@ -725,7 +725,7 @@ class ProcessTest(CoverageTest):
         assert "Goodbye!" in out
 
         msg = (
-            "Coverage.py warning: "
+            "CoverageWarning: "
             "Already imported a file that will be measured: {} "
             "(already-imported)").format(goodbye_path)
         assert msg in out
@@ -815,10 +815,14 @@ class ProcessTest(CoverageTest):
                 inst.save()
             """)
         out = self.run_command("python run_twice.py")
+        # Remove the file location and source line from the warning.
+        out = re.sub(r"(?m)^[\\/\w.:~_-]+:\d+: CoverageWarning: ", "f:d: CoverageWarning: ", out)
+        out = re.sub(r"(?m)^\s+self.warn.*$\n", "", out)
+        print("out:", repr(out))
         expected = (
             "Run 1\n" +
             "Run 2\n" +
-            "Coverage.py warning: Module foo was previously imported, but not measured " +
+            "f:d: CoverageWarning: Module foo was previously imported, but not measured " +
             "(module-not-measured)\n"
         )
         assert expected == out
@@ -920,7 +924,7 @@ class EnvironmentTest(CoverageTest):
     def test_coverage_run_dashm_superset_of_doubledashsource(self):
         """Edge case: --source foo -m foo.bar"""
         # Ugh: without this config file, we'll get a warning about
-        #   Coverage.py warning: Module process_test was previously imported,
+        #   CoverageWarning: Module process_test was previously imported,
         #   but not measured (module-not-measured)
         #
         # This is because process_test/__init__.py is imported while looking
diff --git a/tests/test_summary.py b/tests/test_summary.py
index a6384c46..b71921c7 100644
--- a/tests/test_summary.py
+++ b/tests/test_summary.py
@@ -595,12 +595,13 @@ class SummaryTest(UsingModulesMixin, CoverageTest):
         self.make_file("mycode.py", "This isn't python at all!")
         report = self.report_from_command("coverage report -i mycode.py")
 
-        # Coverage.py warning: Couldn't parse Python file blah_blah/mycode.py (couldnt-parse)
+        # CoverageWarning: Couldn't parse Python file blah_blah/mycode.py (couldnt-parse)
+        #    (source line)
         # Name     Stmts   Miss  Cover
         # ----------------------------
         # No data to report.
 
-        assert self.line_count(report) == 4
+        assert self.line_count(report) == 5
         assert 'No data to report.' in report
         assert '(couldnt-parse)' in report
 
diff --git a/tests/test_testing.py b/tests/test_testing.py
index 3a563efe..7219ff0b 100644
--- a/tests/test_testing.py
+++ b/tests/test_testing.py
@@ -7,16 +7,18 @@ import datetime
 import os
 import re
 import sys
+import warnings
 
 import pytest
 
 import coverage
 from coverage import tomlconfig
+from coverage.exceptions import CoverageWarning
 from coverage.files import actual_path
 
 from tests.coveragetest import CoverageTest
 from tests.helpers import (
-    arcs_to_arcz_repr, arcz_to_arcs, assert_count_equal,
+    arcs_to_arcz_repr, arcz_to_arcs, assert_count_equal, assert_coverage_warnings,
     CheckUniqueFilenames, re_lines, re_line, without_module,
 )
 
@@ -383,3 +385,57 @@ class ArczTest(CoverageTest):
     ])
     def test_arcs_to_arcz_repr(self, arcs, arcz_repr):
         assert arcs_to_arcz_repr(arcs) == arcz_repr
+
+
+class AssertCoverageWarningsTest(CoverageTest):
+    """Tests of assert_coverage_warnings"""
+
+    def test_one_warning(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("Hello there", category=CoverageWarning)
+        assert_coverage_warnings(warns, "Hello there")
+
+    def test_many_warnings(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("The first", category=CoverageWarning)
+            warnings.warn("The second", category=CoverageWarning)
+            warnings.warn("The third", category=CoverageWarning)
+        assert_coverage_warnings(warns, "The first", "The second", "The third")
+
+    def test_wrong_type(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("Not ours", category=Warning)
+        with pytest.raises(AssertionError):
+            assert_coverage_warnings(warns, "Not ours")
+
+    def test_wrong_message(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("Goodbye", category=CoverageWarning)
+        with pytest.raises(AssertionError):
+            assert_coverage_warnings(warns, "Hello there")
+
+    def test_wrong_number_too_many(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("The first", category=CoverageWarning)
+            warnings.warn("The second", category=CoverageWarning)
+        with pytest.raises(AssertionError):
+            assert_coverage_warnings(warns, "The first", "The second", "The third")
+
+    def test_wrong_number_too_few(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("The first", category=CoverageWarning)
+            warnings.warn("The second", category=CoverageWarning)
+            warnings.warn("The third", category=CoverageWarning)
+        with pytest.raises(AssertionError):
+            assert_coverage_warnings(warns, "The first", "The second")
+
+    def test_regex_matches(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("The first", category=CoverageWarning)
+        assert_coverage_warnings(warns, re.compile("f?rst"))
+
+    def test_regex_doesnt_match(self):
+        with pytest.warns(Warning) as warns:
+            warnings.warn("The first", category=CoverageWarning)
+        with pytest.raises(AssertionError):
+            assert_coverage_warnings(warns, re.compile("second"))
diff --git a/tests/test_xml.py b/tests/test_xml.py
index 9c6cfb58..a03257a2 100644
--- a/tests/test_xml.py
+++ b/tests/test_xml.py
@@ -16,7 +16,7 @@ from coverage.misc import import_local_file
 
 from tests.coveragetest import CoverageTest
 from tests.goldtest import compare, gold_path
-from tests.helpers import change_dir
+from tests.helpers import assert_coverage_warnings, change_dir
 
 
 class XmlTestHelpers(CoverageTest):
@@ -213,7 +213,12 @@ class XmlReportTest(XmlTestHelpers, CoverageTest):
         mod_foo = import_local_file("foo", "src/main/foo.py")                   # pragma: nested
         mod_bar = import_local_file("bar", "also/over/there/bar.py")            # pragma: nested
         cov.stop()                                                              # pragma: nested
-        cov.xml_report([mod_foo, mod_bar])
+        with pytest.warns(Warning) as warns:
+            cov.xml_report([mod_foo, mod_bar])
+        assert_coverage_warnings(
+            warns,
+            "Module not/really was never imported. (module-not-imported)",
+            )
         dom = ElementTree.parse("coverage.xml")
 
         self.assert_source(dom, "src/main")