diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/gold/testing/gettysburg.txt | 4 | ||||
| -rw-r--r-- | tests/goldtest.py | 63 | ||||
| -rw-r--r-- | tests/test_goldtest.py | 85 |
3 files changed, 108 insertions, 44 deletions
diff --git a/tests/gold/testing/gettysburg.txt b/tests/gold/testing/gettysburg.txt new file mode 100644 index 00000000..e759ba87 --- /dev/null +++ b/tests/gold/testing/gettysburg.txt @@ -0,0 +1,4 @@ +Four score and seven years ago our fathers brought forth upon this continent, a +new nation, conceived in Liberty, and dedicated to the proposition that all men +are created equal. +11/19/1863, Gettysburg, Pennsylvania diff --git a/tests/goldtest.py b/tests/goldtest.py index 57fa5c06..4f3f882a 100644 --- a/tests/goldtest.py +++ b/tests/goldtest.py @@ -9,10 +9,10 @@ import fnmatch import os import os.path import re -import sys import xml.etree.ElementTree from tests.coveragetest import TESTS_DIR +from tests.helpers import os_sep def gold_path(path): @@ -20,58 +20,34 @@ def gold_path(path): return os.path.join(TESTS_DIR, "gold", path) -def versioned_directory(d): - """Find a subdirectory of d specific to the Python version. - For example, on Python 3.6.4 rc 1, it returns the first of these - directories that exists:: - d/3.6.4.candidate.1 - d/3.6.4.candidate - d/3.6.4 - d/3.6 - d/3 - d - Returns: a string, the path to an existing directory. - """ - ver_parts = list(map(str, sys.version_info)) - for nparts in range(len(ver_parts), -1, -1): - version = ".".join(ver_parts[:nparts]) - subdir = os.path.join(d, version) - if os.path.exists(subdir): - return subdir - raise Exception(f"Directory missing: {d}") # pragma: only failure - - def compare( expected_dir, actual_dir, file_pattern=None, actual_extra=False, scrubs=None, ): """Compare files matching `file_pattern` in `expected_dir` and `actual_dir`. - A version-specific subdirectory of `expected_dir` will be used if - it exists. - `actual_extra` true means `actual_dir` can have extra files in it without triggering an assertion. `scrubs` is a list of pairs: regexes to find and replace to scrub the files of unimportant differences. - An assertion will be raised if the directories fail one of their - matches. + If a comparison fails, a message will be written to stdout, the original + unscrubbed output of the test will be written to an "/actual/" directory + alongside the "/gold/" directory, and an assertion will be raised. """ __tracebackhide__ = True # pytest, please don't show me this function. - - expected_dir = versioned_directory(expected_dir) + assert os_sep("/gold/") in expected_dir dc = filecmp.dircmp(expected_dir, actual_dir) diff_files = fnmatch_list(dc.diff_files, file_pattern) expected_only = fnmatch_list(dc.left_only, file_pattern) actual_only = fnmatch_list(dc.right_only, file_pattern) - def save_mismatch(f): # pragma: only failure + def save_mismatch(f): """Save a mismatched result to tests/actual.""" - save_path = expected_dir.replace("/gold/", "/actual/") + save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/")) os.makedirs(save_path, exist_ok=True) with open(os.path.join(save_path, f), "w") as savef: with open(os.path.join(actual_dir, f)) as readf: @@ -82,7 +58,6 @@ def compare( # ourselves. text_diff = [] for f in diff_files: - expected_file = os.path.join(expected_dir, f) with open(expected_file) as fobj: expected = fobj.read() @@ -98,13 +73,13 @@ def compare( if scrubs: expected = scrub(expected, scrubs) actual = scrub(actual, scrubs) - if expected != actual: # pragma: only failure + if expected != actual: text_diff.append(f'{expected_file} != {actual_file}') expected = expected.splitlines() actual = actual.splitlines() - print(f":::: diff {expected_file!r} and {actual_file!r}") + print(f":::: diff '{expected_file}' and '{actual_file}'") print("\n".join(difflib.Differ().compare(expected, actual))) - print(f":::: end diff {expected_file!r} and {actual_file!r}") + print(f":::: end diff '{expected_file}' and '{actual_file}'") save_mismatch(f) if not actual_extra: # pragma: only failure @@ -118,15 +93,6 @@ def compare( assert not actual_only, f"Files in {actual_dir} only: {actual_only}" -def canonicalize_xml(xtext): - """Canonicalize some XML text.""" - root = xml.etree.ElementTree.fromstring(xtext) - for node in root.iter(): - node.attrib = dict(sorted(node.items())) - xtext = xml.etree.ElementTree.tostring(root) - return xtext.decode("utf-8") - - def contains(filename, *strlist): """Check that the file contains all of a list of strings. @@ -190,6 +156,15 @@ def doesnt_contain(filename, *strlist): # Helpers +def canonicalize_xml(xtext): + """Canonicalize some XML text.""" + root = xml.etree.ElementTree.fromstring(xtext) + for node in root.iter(): + node.attrib = dict(sorted(node.items())) + xtext = xml.etree.ElementTree.tostring(root) + return xtext.decode("utf-8") + + def fnmatch_list(files, file_pattern): """Filter the list of `files` to only those that match `file_pattern`. If `file_pattern` is None, then return the entire list of files. diff --git a/tests/test_goldtest.py b/tests/test_goldtest.py new file mode 100644 index 00000000..fcd41579 --- /dev/null +++ b/tests/test_goldtest.py @@ -0,0 +1,85 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt + +"""Tests of the helpers in goldtest.py""" + +import os.path +import re + +import pytest + +from tests.coveragetest import CoverageTest, TESTS_DIR +from tests.goldtest import compare, gold_path +from tests.helpers import os_sep, re_line, remove_tree + +GOOD_GETTY = """\ +Four score and seven years ago our fathers brought forth upon this continent, a +new nation, conceived in Liberty, and dedicated to the proposition that all men +are created equal. +11/19/9999, Gettysburg, Pennsylvania +""" + +BAD_GETTY = """\ +Five score and seven years ago our fathers brought forth upon this continent, a +new nation, conceived in Liberty, and dedicated to the proposition that all men +are created equal. +333/4444/55555, Gettysburg, Pennsylvania +""" + +SCRUBS = [ + # Numbers don't matter when comparing. + (r'\d+', 'D'), +] + +ACTUAL_DIR = os.path.join(TESTS_DIR, "actual/testing") +ACTUAL_GETTY_FILE = os.path.join(ACTUAL_DIR, "gettysburg.txt") + +GOLD_PATH_RE = re.escape(os_sep("/tests/gold/testing/gettysburg.txt")) +OUT_PATH_RE = re.escape(os_sep("out/gettysburg.txt")) + +class CompareTest(CoverageTest): + """Tests of goldtest.py:compare()""" + + def setup_test(self): + super().setup_test() + self.addCleanup(remove_tree, ACTUAL_DIR) + + def test_compare_good(self): + self.make_file("out/gettysburg.txt", GOOD_GETTY) + compare(gold_path("testing"), "out", scrubs=SCRUBS) + self.assert_doesnt_exist(ACTUAL_GETTY_FILE) + + def test_compare_bad(self): + self.make_file("out/gettysburg.txt", BAD_GETTY) + + # compare() raises an assertion. + msg = rf"Files differ: .*{GOLD_PATH_RE} != {OUT_PATH_RE}" + with pytest.raises(AssertionError, match=msg): + compare(gold_path("testing"), "out", scrubs=SCRUBS) + + # Stdout has a description of the diff. The diff shows the scrubbed content. + stdout = self.stdout() + print(stdout) + assert "- Four score" in stdout + assert "+ Five score" in stdout + assert re_line(stdout, rf"^:::: diff '.*{GOLD_PATH_RE}' and '{OUT_PATH_RE}'") + assert re_line(stdout, rf"^:::: end diff '.*{GOLD_PATH_RE}' and '{OUT_PATH_RE}'") + assert " D/D/D, Gettysburg, Pennsylvania" in stdout + + # The actual file was saved. + with open(ACTUAL_GETTY_FILE) as f: + saved = f.read() + assert saved == BAD_GETTY + + def test_compare_good_needs_scrubs(self): + # Comparing the "good" result without scrubbing the variable parts will fail. + self.make_file("out/gettysburg.txt", GOOD_GETTY) + + # compare() raises an assertion. + msg = rf"Files differ: .*{GOLD_PATH_RE} != {OUT_PATH_RE}" + with pytest.raises(AssertionError, match=msg): + compare(gold_path("testing"), "out") + + stdout = self.stdout() + assert "- 11/19/1863, Gettysburg, Pennsylvania" in stdout + assert "+ 11/19/9999, Gettysburg, Pennsylvania" in stdout |