From 229151e1f399cefadf843a41ee08791fa1aa451a Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 13:56:05 +0100 Subject: Extract out a filter base class that just deals with predicates. --- python/subunit/test_results.py | 164 ++++++++++++++++++++++++----------------- 1 file changed, 96 insertions(+), 68 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index fb7affd..32dff6a 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -286,7 +286,100 @@ def all_true(bools): return True -class TestResultFilter(TestResultDecorator): +class _PredicateFilter(TestResultDecorator): + + def __init__(self, result, predicate): + super(_PredicateFilter, self).__init__(result) + self.decorated = TimeCollapsingDecorator( + TagCollapsingDecorator(self.decorated)) + self.filter_predicate = predicate + # The current test (for filtering tags) + self._current_test = None + # Has the current test been filtered (for outputting test tags) + self._current_test_filtered = None + # Calls to this result that we don't know whether to forward on yet. + self._buffered_calls = [] + + def addError(self, test, err=None, details=None): + if (self.filter_predicate(test, 'error', err, details)): + self._buffered_calls.append( + ('addError', [test, err], {'details': details})) + else: + self._filtered() + + def addFailure(self, test, err=None, details=None): + if (self.filter_predicate(test, 'failure', err, details)): + self._buffered_calls.append( + ('addFailure', [test, err], {'details': details})) + else: + self._filtered() + + def addSkip(self, test, reason=None, details=None): + if (self.filter_predicate(test, 'skip', reason, details)): + self._buffered_calls.append( + ('addSkip', [test, reason], {'details': details})) + else: + self._filtered() + + def addExpectedFailure(self, test, err=None, details=None): + if self.filter_predicate(test, 'expectedfailure', err, details): + self._buffered_calls.append( + ('addExpectedFailure', [test, err], {'details': details})) + else: + self._filtered() + + def addUnexpectedSuccess(self, test, details=None): + self._buffered_calls.append( + ('addUnexpectedSuccess', [test], {'details': details})) + + def addSuccess(self, test, details=None): + if (self.filter_predicate(test, 'success', None, details)): + self._buffered_calls.append( + ('addSuccess', [test], {'details': details})) + else: + self._filtered() + + def _filtered(self): + self._current_test_filtered = True + + def startTest(self, test): + """Start a test. + + Not directly passed to the client, but used for handling of tags + correctly. + """ + self._current_test = test + self._current_test_filtered = False + self._buffered_calls.append(('startTest', [test], {})) + + def stopTest(self, test): + """Stop a test. + + Not directly passed to the client, but used for handling of tags + correctly. + """ + if not self._current_test_filtered: + # Tags to output for this test. + for method, args, kwargs in self._buffered_calls: + getattr(self.decorated, method)(*args, **kwargs) + self.decorated.stopTest(test) + self._current_test = None + self._current_test_filtered = None + self._buffered_calls = [] + + def time(self, a_time): + if self._current_test is not None: + self._buffered_calls.append(('time', [a_time], {})) + else: + return self.decorated.time(a_time) + + def id_to_orig_id(self, id): + if id.startswith("subunit.RemotedTestCase."): + return id[len("subunit.RemotedTestCase."):] + return id + + +class TestResultFilter(_PredicateFilter): """A pyunit TestResult interface implementation which filters tests. Tests that pass the filter are handed on to another TestResult instance @@ -316,9 +409,6 @@ class TestResultFilter(TestResultDecorator): :param fixup_expected_failures: Set of test ids to consider known failing. """ - super(TestResultFilter, self).__init__(result) - self.decorated = TimeCollapsingDecorator( - TagCollapsingDecorator(self.decorated)) predicates = [] if filter_error: predicates.append(lambda t, outcome, e, d: outcome != 'error') @@ -332,15 +422,10 @@ class TestResultFilter(TestResultDecorator): predicates.append(lambda t, outcome, e, d: outcome != 'expectedfailure') if filter_predicate is not None: predicates.append(filter_predicate) - self.filter_predicate = ( + predicate = ( lambda test, outcome, err, details: all_true(p(test, outcome, err, details) for p in predicates)) - # The current test (for filtering tags) - self._current_test = None - # Has the current test been filtered (for outputting test tags) - self._current_test_filtered = None - # Calls to this result that we don't know whether to forward on yet. - self._buffered_calls = [] + super(TestResultFilter, self).__init__(result, predicate) if fixup_expected_failures is None: self._fixup_expected_failures = frozenset() else: @@ -368,13 +453,6 @@ class TestResultFilter(TestResultDecorator): else: self._filtered() - def addSkip(self, test, reason=None, details=None): - if (self.filter_predicate(test, 'skip', reason, details)): - self._buffered_calls.append( - ('addSkip', [test, reason], {'details': details})) - else: - self._filtered() - def addSuccess(self, test, details=None): if (self.filter_predicate(test, 'success', None, details)): if self._failure_expected(test): @@ -386,59 +464,9 @@ class TestResultFilter(TestResultDecorator): else: self._filtered() - def addExpectedFailure(self, test, err=None, details=None): - if self.filter_predicate(test, 'expectedfailure', err, details): - self._buffered_calls.append( - ('addExpectedFailure', [test, err], {'details': details})) - else: - self._filtered() - - def addUnexpectedSuccess(self, test, details=None): - self._buffered_calls.append( - ('addUnexpectedSuccess', [test], {'details': details})) - - def _filtered(self): - self._current_test_filtered = True - def _failure_expected(self, test): return (test.id() in self._fixup_expected_failures) - def startTest(self, test): - """Start a test. - - Not directly passed to the client, but used for handling of tags - correctly. - """ - self._current_test = test - self._current_test_filtered = False - self._buffered_calls.append(('startTest', [test], {})) - - def stopTest(self, test): - """Stop a test. - - Not directly passed to the client, but used for handling of tags - correctly. - """ - if not self._current_test_filtered: - # Tags to output for this test. - for method, args, kwargs in self._buffered_calls: - getattr(self.decorated, method)(*args, **kwargs) - self.decorated.stopTest(test) - self._current_test = None - self._current_test_filtered = None - self._buffered_calls = [] - - def time(self, a_time): - if self._current_test is not None: - self._buffered_calls.append(('time', [a_time], {})) - else: - return self.decorated.time(a_time) - - def id_to_orig_id(self, id): - if id.startswith("subunit.RemotedTestCase."): - return id[len("subunit.RemotedTestCase."):] - return id - class TestIdPrintingResult(testtools.TestResult): -- cgit v1.2.1 From ffc3ec6be067481ca601cf3cf604a548c6279ea7 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 13:59:31 +0100 Subject: Factor out the "fixup expected failures" thing so they look more like result transformers. --- python/subunit/test_results.py | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 32dff6a..7acd5ea 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -432,37 +432,24 @@ class TestResultFilter(_PredicateFilter): self._fixup_expected_failures = fixup_expected_failures def addError(self, test, err=None, details=None): - if (self.filter_predicate(test, 'error', err, details)): - if self._failure_expected(test): - self._buffered_calls.append( - ('addExpectedFailure', [test, err], {'details': details})) - else: - self._buffered_calls.append( - ('addError', [test, err], {'details': details})) + if self._failure_expected(test): + self.addExpectedFailure(test, err=err, details=details) else: - self._filtered() + super(TestResultFilter, self).addError( + test, err=err, details=details) def addFailure(self, test, err=None, details=None): - if (self.filter_predicate(test, 'failure', err, details)): - if self._failure_expected(test): - self._buffered_calls.append( - ('addExpectedFailure', [test, err], {'details': details})) - else: - self._buffered_calls.append( - ('addFailure', [test, err], {'details': details})) + if self._failure_expected(test): + self.addExpectedFailure(test, err=err, details=details) else: - self._filtered() + super(TestResultFilter, self).addFailure( + test, err=err, details=details) def addSuccess(self, test, details=None): - if (self.filter_predicate(test, 'success', None, details)): - if self._failure_expected(test): - self._buffered_calls.append( - ('addUnexpectedSuccess', [test], {'details': details})) - else: - self._buffered_calls.append( - ('addSuccess', [test], {'details': details})) + if self._failure_expected(test): + self.addUnexpectedSuccess(test, details=details) else: - self._filtered() + super(TestResultFilter, self).addSuccess(test, details=details) def _failure_expected(self, test): return (test.id() in self._fixup_expected_failures) -- cgit v1.2.1 From 6f3bfe4a750ed5c34a1e61705593d935c0cff38d Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 14:12:14 +0100 Subject: Flakes --- python/subunit/tests/test_subunit_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/subunit') diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 0675484..83b9240 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -21,7 +21,7 @@ from subunit import iso8601 import unittest from testtools import TestCase -from testtools.compat import _b, BytesIO, StringIO +from testtools.compat import _b, BytesIO from testtools.testresult.doubles import ExtendedTestResult import subunit -- cgit v1.2.1 From bd5107a642a7b704661e492566e588e944669a25 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 14:12:20 +0100 Subject: A layer of abstraction that can help us. --- python/subunit/test_results.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 7acd5ea..92e0310 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -292,7 +292,7 @@ class _PredicateFilter(TestResultDecorator): super(_PredicateFilter, self).__init__(result) self.decorated = TimeCollapsingDecorator( TagCollapsingDecorator(self.decorated)) - self.filter_predicate = predicate + self._predicate = predicate # The current test (for filtering tags) self._current_test = None # Has the current test been filtered (for outputting test tags) @@ -300,6 +300,9 @@ class _PredicateFilter(TestResultDecorator): # Calls to this result that we don't know whether to forward on yet. self._buffered_calls = [] + def filter_predicate(self, test, outcome, error, details): + return self._predicate(test, outcome, error, details) + def addError(self, test, err=None, details=None): if (self.filter_predicate(test, 'error', err, details)): self._buffered_calls.append( -- cgit v1.2.1 From 8ff7d39c1e1e247d860053026d6fd0867379adb0 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 14:26:44 +0100 Subject: Allow the predicate to filter tags. --- python/subunit/test_results.py | 13 ++++++++++++- python/subunit/tests/test_test_results.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 92e0310..0c6fbd5 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -80,6 +80,10 @@ class TestResultDecorator(object): def wasSuccessful(self): return self.decorated.wasSuccessful() + @property + def current_tags(self): + return self.decorated.current_tags + @property def shouldStop(self): return self.decorated.shouldStop @@ -301,7 +305,14 @@ class _PredicateFilter(TestResultDecorator): self._buffered_calls = [] def filter_predicate(self, test, outcome, error, details): - return self._predicate(test, outcome, error, details) + # XXX: ExtendedToOriginalDecorator doesn't properly wrap current_tags. + # https://bugs.launchpad.net/testtools/+bug/978027 + tags = getattr(self.decorated, 'current_tags', frozenset()) + # 0.0.7 and earlier did not support the 'tags' parameter. + try: + return self._predicate(test, outcome, error, details, tags) + except TypeError: + return self._predicate(test, outcome, error, details) def addError(self, test, err=None, details=None): if (self.filter_predicate(test, 'error', err, details)): diff --git a/python/subunit/tests/test_test_results.py b/python/subunit/tests/test_test_results.py index 6beb57a..2bec7e3 100644 --- a/python/subunit/tests/test_test_results.py +++ b/python/subunit/tests/test_test_results.py @@ -56,6 +56,16 @@ class AssertBeforeTestResult(LoggingDecorator): super(AssertBeforeTestResult, self)._before_event() +class TestTestResultDecorator(unittest.TestCase): + + def test_current_tags(self): + result = ExtendedTestResult() + decorator = subunit.test_results.TestResultDecorator(result) + decorator.tags(set('foo'), set()) + self.assertEqual(set('foo'), decorator.current_tags) + self.assertEqual(decorator.current_tags, result.current_tags) + + class TimeCapturingResult(unittest.TestResult): def __init__(self): -- cgit v1.2.1 From 3cbc2211d91877f2c24c9bed2f50c5a68cfa9deb Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 14:39:10 +0100 Subject: Make sure all of our predicates support tags. --- python/subunit/test_results.py | 35 ++++++++++++++++++----------- python/subunit/tests/test_subunit_filter.py | 14 ++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 0c6fbd5..25c5c77 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -308,11 +308,7 @@ class _PredicateFilter(TestResultDecorator): # XXX: ExtendedToOriginalDecorator doesn't properly wrap current_tags. # https://bugs.launchpad.net/testtools/+bug/978027 tags = getattr(self.decorated, 'current_tags', frozenset()) - # 0.0.7 and earlier did not support the 'tags' parameter. - try: - return self._predicate(test, outcome, error, details, tags) - except TypeError: - return self._predicate(test, outcome, error, details) + return self._predicate(test, outcome, error, details, tags) def addError(self, test, err=None, details=None): if (self.filter_predicate(test, 'error', err, details)): @@ -425,20 +421,33 @@ class TestResultFilter(_PredicateFilter): """ predicates = [] if filter_error: - predicates.append(lambda t, outcome, e, d: outcome != 'error') + predicates.append( + lambda t, outcome, e, d, tags: outcome != 'error') if filter_failure: - predicates.append(lambda t, outcome, e, d: outcome != 'failure') + predicates.append( + lambda t, outcome, e, d, tags: outcome != 'failure') if filter_success: - predicates.append(lambda t, outcome, e, d: outcome != 'success') + predicates.append( + lambda t, outcome, e, d, tags: outcome != 'success') if filter_skip: - predicates.append(lambda t, outcome, e, d: outcome != 'skip') + predicates.append( + lambda t, outcome, e, d, tags: outcome != 'skip') if filter_xfail: - predicates.append(lambda t, outcome, e, d: outcome != 'expectedfailure') + predicates.append( + lambda t, outcome, e, d, tags: outcome != 'expectedfailure') if filter_predicate is not None: - predicates.append(filter_predicate) + def compat(test, outcome, error, details, tags): + # 0.0.7 and earlier did not support the 'tags' parameter. + try: + return filter_predicate( + test, outcome, error, details, tags) + except TypeError: + return filter_predicate(test, outcome, error, details) + predicates.append(compat) predicate = ( - lambda test, outcome, err, details: - all_true(p(test, outcome, err, details) for p in predicates)) + lambda test, outcome, err, details, tags: + all_true(p(test, outcome, err, details, tags) + for p in predicates)) super(TestResultFilter, self).__init__(result, predicate) if fixup_expected_failures is None: self._fixup_expected_failures = frozenset() diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 83b9240..5bbf581 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -151,6 +151,8 @@ xfail todo def test_filter_predicate(self): """You can filter by predicate callbacks""" + # 0.0.7 and earlier did not support the 'tags' parameter, so we need + # to test that we still support behaviour without it. filtered_result = unittest.TestResult() def filter_cb(test, outcome, err, details): return outcome == 'success' @@ -161,6 +163,18 @@ xfail todo # Only success should pass self.assertEqual(1, filtered_result.testsRun) + def test_filter_predicate_with_tags(self): + """You can filter by predicate callbacks that accept tags""" + filtered_result = unittest.TestResult() + def filter_cb(test, outcome, err, details, tags): + return outcome == 'success' + result_filter = TestResultFilter(filtered_result, + filter_predicate=filter_cb, + filter_success=False) + self.run_tests(result_filter) + # Only success should pass + self.assertEqual(1, filtered_result.testsRun) + def test_time_ordering_preserved(self): # Passing a subunit stream through TestResultFilter preserves the # relative ordering of 'time' directives and any other subunit -- cgit v1.2.1 From 60d01e6fbb76c9e1ae5bcecd0a4ae6e96dfc3174 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 14:41:45 +0100 Subject: Composition is better than inheritance. --- python/subunit/test_results.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 25c5c77..6c74ec3 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -389,7 +389,7 @@ class _PredicateFilter(TestResultDecorator): return id -class TestResultFilter(_PredicateFilter): +class TestResultFilter(TestResultDecorator): """A pyunit TestResult interface implementation which filters tests. Tests that pass the filter are handed on to another TestResult instance @@ -448,7 +448,8 @@ class TestResultFilter(_PredicateFilter): lambda test, outcome, err, details, tags: all_true(p(test, outcome, err, details, tags) for p in predicates)) - super(TestResultFilter, self).__init__(result, predicate) + super(TestResultFilter, self).__init__( + _PredicateFilter(result, predicate)) if fixup_expected_failures is None: self._fixup_expected_failures = frozenset() else: -- cgit v1.2.1 From 6de2212472caa1e7abab5b0a517aebbaf483141e Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 14:54:40 +0100 Subject: Extract a helper.p --- python/subunit/test_results.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 6c74ec3..3e597ef 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -20,6 +20,7 @@ import csv import datetime import testtools +from testtools.compat import all from testtools.content import ( text_content, TracebackContent, @@ -282,12 +283,10 @@ class TimeCollapsingDecorator(HookedTestResultDecorator): self._last_received_time = a_time -def all_true(bools): - """Return True if all of 'bools' are True. False otherwise.""" - for b in bools: - if not b: - return False - return True +def and_predicates(predicates): + """Return a predicate that is true iff all predicates are true.""" + # XXX: Should probably be in testtools to be better used by matchers. jml + return lambda *args, **kwargs: all(p(*args, **kwargs) for p in predicates) class _PredicateFilter(TestResultDecorator): @@ -444,10 +443,7 @@ class TestResultFilter(TestResultDecorator): except TypeError: return filter_predicate(test, outcome, error, details) predicates.append(compat) - predicate = ( - lambda test, outcome, err, details, tags: - all_true(p(test, outcome, err, details, tags) - for p in predicates)) + predicate = and_predicates(predicates) super(TestResultFilter, self).__init__( _PredicateFilter(result, predicate)) if fixup_expected_failures is None: -- cgit v1.2.1 From 8141fda3abe0ba97903dca15a40f37dfa4eebcb7 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 10 Apr 2012 16:10:34 +0100 Subject: Add tests that exercise the subunit-filter filter. --- python/subunit/tests/test_subunit_filter.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'python/subunit') diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 5bbf581..1cae791 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -17,6 +17,9 @@ """Tests for subunit.TestResultFilter.""" from datetime import datetime +import os +import subprocess +import sys from subunit import iso8601 import unittest @@ -216,6 +219,58 @@ xfail todo ('stopTest', foo), ], result._events) +class TestFilterCommand(TestCase): + + example_subunit_stream = _b("""\ +tags: global +test passed +success passed +test failed +tags: local +failure failed +test error +error error [ +error details +] +test skipped +skip skipped +test todo +xfail todo +""") + + def run_command(self, args, stream): + root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + script_path = os.path.join(root, 'filters', 'subunit-filter') + command = [sys.executable, script_path] + list(args) + ps = subprocess.Popen( + command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = ps.communicate(stream) + if ps.returncode != 0: + raise RuntimeError("%s failed: %s" % (command, err)) + return out + + def to_events(self, stream): + test = subunit.ProtocolTestCase(BytesIO(stream)) + result = ExtendedTestResult() + test.run(result) + return result._events + + def test_default(self): + output = self.run_command([], ( + "test: foo\n" + "skip: foo\n" + )) + events = self.to_events(output) + foo = subunit.RemotedTestCase('foo') + self.assertEqual( + [('startTest', foo), + ('addSkip', foo, {}), + ('stopTest', foo)], + events) + + def test_suite(): loader = subunit.tests.TestUtil.TestLoader() result = loader.loadTestsFromName(__name__) -- cgit v1.2.1 From 994467ea854c08498baac63f01a653a240514526 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 13 Apr 2012 00:12:32 +0100 Subject: Progress, of a sort. --- python/subunit/test_results.py | 22 +++++++++++++++--- python/subunit/tests/test_subunit_filter.py | 35 ++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 4 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 3e597ef..deaea1b 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -289,6 +289,22 @@ def and_predicates(predicates): return lambda *args, **kwargs: all(p(*args, **kwargs) for p in predicates) +def _make_tag_filter(with_tags, without_tags): + """Make a callback that checks tests against tags.""" + + with_tags = with_tags and set(with_tags) or None + without_tags = without_tags and set(without_tags) or None + + def check_tags(test, outcome, err, details, tags): + if with_tags and not with_tags <= tags: + return False + if without_tags and bool(without_tags & tags): + return False + return True + + return check_tags + + class _PredicateFilter(TestResultDecorator): def __init__(self, result, predicate): @@ -306,8 +322,8 @@ class _PredicateFilter(TestResultDecorator): def filter_predicate(self, test, outcome, error, details): # XXX: ExtendedToOriginalDecorator doesn't properly wrap current_tags. # https://bugs.launchpad.net/testtools/+bug/978027 - tags = getattr(self.decorated, 'current_tags', frozenset()) - return self._predicate(test, outcome, error, details, tags) + return self._predicate( + test, outcome, error, details, self.current_tags) def addError(self, test, err=None, details=None): if (self.filter_predicate(test, 'error', err, details)): @@ -538,7 +554,7 @@ class TestIdPrintingResult(testtools.TestResult): class TestByTestResult(testtools.TestResult): """Call something every time a test completes.""" - # XXX: Arguably belongs in testtools. +# XXX: Arguably belongs in testtools. def __init__(self, on_test): """Construct a ``TestByTestResult``. diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 1cae791..d5f204a 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -28,7 +28,7 @@ from testtools.compat import _b, BytesIO from testtools.testresult.doubles import ExtendedTestResult import subunit -from subunit.test_results import TestResultFilter +from subunit.test_results import _make_tag_filter, TestResultFilter class TestTestResultFilter(TestCase): @@ -80,6 +80,22 @@ xfail todo filtered_result.failures]) self.assertEqual(4, filtered_result.testsRun) + def test_tag_filter(self): + tag_filter = _make_tag_filter(['global'], ['local']) + result = ExtendedTestResult() + result_filter = TestResultFilter( + result, filter_success=False, filter_predicate=tag_filter) + self.run_tests(result_filter) + test = subunit.RemotedTestCase('passed') + self.assertEquals( + [('tags', set(['global']), set()), + ('startTest', test), + ('addSuccess', test), + ('stopTest', test), + ('tags', set(['local']), set()), + ], + result._events) + def test_exclude_errors(self): filtered_result = unittest.TestResult() result_filter = TestResultFilter(filtered_result, filter_error=True) @@ -270,6 +286,23 @@ xfail todo ('stopTest', foo)], events) + def test_tags(self): + output = self.run_command(['-s', '--with-tag', 'a'], ( + "tags: a\n" + "test: foo\n" + "success: foo\n" + "tags: -a\n" + "test: bar\n" + "success: bar\n" + )) + events = self.to_events(output) + foo = subunit.RemotedTestCase('foo') + self.assertEqual( + [('startTest', foo), + ('addSuccess', foo), + ('stopTest', foo)], + events) + def test_suite(): loader = subunit.tests.TestUtil.TestLoader() -- cgit v1.2.1 From dcf88f6c47a42c791ddf257ee4f6050e943ba906 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Thu, 19 Apr 2012 13:57:29 +0100 Subject: Comments. --- python/subunit/test_results.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index deaea1b..ce8f67a 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -40,6 +40,9 @@ class TestResultDecorator(object): or features by degrading them. """ + # XXX: Since lp:testtools r250, this is in testtools. Once it's released, + # we should gut this and just use that. + def __init__(self, decorated): """Create a TestResultDecorator forwarding to decorated.""" # Make every decorator degrade gracefully. -- cgit v1.2.1 From 87338c132fb605881db065c20b9aa2f293946e8e Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Thu, 19 Apr 2012 19:13:36 +0100 Subject: don't rely on current_tags, implement it ourselves. --- python/subunit/test_results.py | 17 +++++++++++------ python/subunit/tests/test_subunit_filter.py | 8 +++++--- python/subunit/tests/test_test_results.py | 10 ---------- 3 files changed, 16 insertions(+), 19 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index ce8f67a..b6e9be8 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -84,10 +84,6 @@ class TestResultDecorator(object): def wasSuccessful(self): return self.decorated.wasSuccessful() - @property - def current_tags(self): - return self.decorated.current_tags - @property def shouldStop(self): return self.decorated.shouldStop @@ -315,6 +311,7 @@ class _PredicateFilter(TestResultDecorator): self.decorated = TimeCollapsingDecorator( TagCollapsingDecorator(self.decorated)) self._predicate = predicate + self._current_tags = set() # The current test (for filtering tags) self._current_test = None # Has the current test been filtered (for outputting test tags) @@ -326,7 +323,7 @@ class _PredicateFilter(TestResultDecorator): # XXX: ExtendedToOriginalDecorator doesn't properly wrap current_tags. # https://bugs.launchpad.net/testtools/+bug/978027 return self._predicate( - test, outcome, error, details, self.current_tags) + test, outcome, error, details, self._current_tags) def addError(self, test, err=None, details=None): if (self.filter_predicate(test, 'error', err, details)): @@ -387,7 +384,6 @@ class _PredicateFilter(TestResultDecorator): correctly. """ if not self._current_test_filtered: - # Tags to output for this test. for method, args, kwargs in self._buffered_calls: getattr(self.decorated, method)(*args, **kwargs) self.decorated.stopTest(test) @@ -395,6 +391,15 @@ class _PredicateFilter(TestResultDecorator): self._current_test_filtered = None self._buffered_calls = [] + def tags(self, new_tags, gone_tags): + new_tags, gone_tags = set(new_tags), set(gone_tags) + self._current_tags.update(new_tags) + self._current_tags.difference_update(gone_tags) + if self._current_test is not None: + self._buffered_calls.append(('tags', [new_tags, gone_tags], {})) + else: + return super(_PredicateFilter, self).tags(new_tags, gone_tags) + def time(self, a_time): if self._current_test is not None: self._buffered_calls.append(('time', [a_time], {})) diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index d5f204a..35d4603 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -92,7 +92,6 @@ xfail todo ('startTest', test), ('addSuccess', test), ('stopTest', test), - ('tags', set(['local']), set()), ], result._events) @@ -298,9 +297,12 @@ xfail todo events = self.to_events(output) foo = subunit.RemotedTestCase('foo') self.assertEqual( - [('startTest', foo), + [('tags', set(['a']), set()), + ('startTest', foo), ('addSuccess', foo), - ('stopTest', foo)], + ('stopTest', foo), + ('tags', set(), set(['a'])), + ], events) diff --git a/python/subunit/tests/test_test_results.py b/python/subunit/tests/test_test_results.py index 2bec7e3..6beb57a 100644 --- a/python/subunit/tests/test_test_results.py +++ b/python/subunit/tests/test_test_results.py @@ -56,16 +56,6 @@ class AssertBeforeTestResult(LoggingDecorator): super(AssertBeforeTestResult, self)._before_event() -class TestTestResultDecorator(unittest.TestCase): - - def test_current_tags(self): - result = ExtendedTestResult() - decorator = subunit.test_results.TestResultDecorator(result) - decorator.tags(set('foo'), set()) - self.assertEqual(set('foo'), decorator.current_tags) - self.assertEqual(decorator.current_tags, result.current_tags) - - class TimeCapturingResult(unittest.TestResult): def __init__(self): -- cgit v1.2.1 From a34add828e0891ea2309141e1358bdbf3ba3fbf4 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 20 Apr 2012 10:54:17 +0100 Subject: Make sure tags are sent before result. --- python/subunit/test_results.py | 15 +++++++-------- python/subunit/tests/test_subunit_filter.py | 1 - python/subunit/tests/test_test_results.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index b6e9be8..dec168d 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -209,7 +209,7 @@ class AutoTimingTestResultDecorator(HookedTestResultDecorator): return self.decorated.time(a_datetime) -class TagCollapsingDecorator(TestResultDecorator): +class TagCollapsingDecorator(HookedTestResultDecorator): """Collapses many 'tags' calls into one where possible.""" def __init__(self, result): @@ -227,16 +227,15 @@ class TagCollapsingDecorator(TestResultDecorator): self._current_test_tags = set(), set() def stopTest(self, test): - """Stop a test. + super(TagCollapsingDecorator, self).stopTest(test) + self._current_test_tags = set(), set() - Not directly passed to the client, but used for handling of tags - correctly. - """ - # Tags to output for this test. + def _before_event(self): + if not self._current_test_tags: + return if self._current_test_tags[0] or self._current_test_tags[1]: self.decorated.tags(*self._current_test_tags) - self.decorated.stopTest(test) - self._current_test_tags = None + self._current_test_tags = set(), set() def tags(self, new_tags, gone_tags): """Handle tag instructions. diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 35d4603..e04090d 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -301,7 +301,6 @@ xfail todo ('startTest', foo), ('addSuccess', foo), ('stopTest', foo), - ('tags', set(), set(['a'])), ], events) diff --git a/python/subunit/tests/test_test_results.py b/python/subunit/tests/test_test_results.py index 6beb57a..d11f482 100644 --- a/python/subunit/tests/test_test_results.py +++ b/python/subunit/tests/test_test_results.py @@ -238,6 +238,25 @@ class TestTagCollapsingDecorator(TestCase): ('stopTest', test)], result._events) + def test_tags_sent_before_result(self): + # Because addSuccess and friends tend to send subunit output + # immediately, and because 'tags:' before a result line means + # something different to 'tags:' after a result line, we need to be + # sure that tags are emitted before 'addSuccess' (or whatever). + result = ExtendedTestResult() + tag_collapser = subunit.test_results.TagCollapsingDecorator(result) + test = subunit.RemotedTestCase('foo') + tag_collapser.startTest(test) + tag_collapser.tags(set(['a']), set()) + tag_collapser.addSuccess(test) + tag_collapser.stopTest(test) + self.assertEquals( + [('startTest', test), + ('tags', set(['a']), set()), + ('addSuccess', test), + ('stopTest', test)], + result._events) + class TestTimeCollapsingDecorator(TestCase): -- cgit v1.2.1 From 59b5ed77506b2ca196b9a454fc032e5897f1ca37 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 20 Apr 2012 10:57:47 +0100 Subject: Properly scope tag collapsing --- python/subunit/test_results.py | 2 +- python/subunit/tests/test_subunit_filter.py | 1 + python/subunit/tests/test_test_results.py | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index dec168d..1988346 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -228,7 +228,7 @@ class TagCollapsingDecorator(HookedTestResultDecorator): def stopTest(self, test): super(TagCollapsingDecorator, self).stopTest(test) - self._current_test_tags = set(), set() + self._current_test_tags = None def _before_event(self): if not self._current_test_tags: diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index e04090d..35d4603 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -301,6 +301,7 @@ xfail todo ('startTest', foo), ('addSuccess', foo), ('stopTest', foo), + ('tags', set(), set(['a'])), ], events) diff --git a/python/subunit/tests/test_test_results.py b/python/subunit/tests/test_test_results.py index d11f482..750958e 100644 --- a/python/subunit/tests/test_test_results.py +++ b/python/subunit/tests/test_test_results.py @@ -208,6 +208,22 @@ class TestTagCollapsingDecorator(TestCase): self.assertEquals( [('tags', set(['a', 'b']), set([]))], result._events) + def test_tags_forwarded_after_tests(self): + test = subunit.RemotedTestCase('foo') + result = ExtendedTestResult() + tag_collapser = subunit.test_results.TagCollapsingDecorator(result) + tag_collapser.startTest(test) + tag_collapser.addSuccess(test) + tag_collapser.stopTest(test) + tag_collapser.tags(set(['a']), set(['b'])) + self.assertEqual( + [('startTest', test), + ('addSuccess', test), + ('stopTest', test), + ('tags', set(['a']), set(['b'])), + ], + result._events) + def test_tags_collapsed_inside_of_tests(self): result = ExtendedTestResult() tag_collapser = subunit.test_results.TagCollapsingDecorator(result) -- cgit v1.2.1 From 9cb6cecd1665dbbaf7508e182565d6bd9b10e812 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 20 Apr 2012 12:36:48 +0100 Subject: Make the integration test include local tags as well. --- python/subunit/tests/test_subunit_filter.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'python/subunit') diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 35d4603..7eca4cd 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -293,15 +293,23 @@ xfail todo "tags: -a\n" "test: bar\n" "success: bar\n" + "test: baz\n" + "tags: a\n" + "success: baz\n" )) events = self.to_events(output) foo = subunit.RemotedTestCase('foo') + baz = subunit.RemotedTestCase('baz') self.assertEqual( [('tags', set(['a']), set()), ('startTest', foo), ('addSuccess', foo), ('stopTest', foo), ('tags', set(), set(['a'])), + ('startTest', baz), + ('tags', set(['a']), set()), + ('addSuccess', baz), + ('stopTest', baz), ], events) -- cgit v1.2.1 From 72054b561629ef3bb03faff0b28bcb9ad4c787c3 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 20 Apr 2012 17:10:17 +0100 Subject: Factor a TagsMixin out of TagCollapsingDecorator --- python/subunit/test_results.py | 69 ++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 29 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index fea3b07..7cce660 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -209,47 +209,44 @@ class AutoTimingTestResultDecorator(HookedTestResultDecorator): return self.decorated.time(a_datetime) -class TagCollapsingDecorator(HookedTestResultDecorator): - """Collapses many 'tags' calls into one where possible.""" +class TagsMixin(object): - def __init__(self, result): - super(TagCollapsingDecorator, self).__init__(result) + def __init__(self): self._clear_tags() def _clear_tags(self): self._global_tags = set(), set() - self._current_test_tags = None - - def _get_current_tags(self): - if self._current_test_tags: - return self._current_test_tags + self._test_tags = None + + def _get_active_tags(self): + global_new, global_gone = self._global_tags + if self._test_tags is None: + return set(global_new) + test_new, test_gone = self._test_tags + return global_new.difference(test_gone).union(test_new) + + def _get_current_scope(self): + if self._test_tags: + return self._test_tags return self._global_tags + def _flush_current_scope(self, tag_receiver): + new_tags, gone_tags = self._get_current_scope() + if new_tags or gone_tags: + tag_receiver.tags(new_tags, gone_tags) + if self._test_tags: + self._test_tags = set(), set() + else: + self._global_tags = set(), set() + def startTestRun(self): - super(TagCollapsingDecorator, self).startTestRun() self._clear_tags() def startTest(self, test): - """Start a test. - - Not directly passed to the client, but used for handling of tags - correctly. - """ - super(TagCollapsingDecorator, self).startTest(test) - self._current_test_tags = set(), set() + self._test_tags = set(), set() def stopTest(self, test): - super(TagCollapsingDecorator, self).stopTest(test) - self._current_test_tags = None - - def _before_event(self): - new_tags, gone_tags = self._get_current_tags() - if new_tags or gone_tags: - self.decorated.tags(new_tags, gone_tags) - if self._current_test_tags: - self._current_test_tags = set(), set() - else: - self._global_tags = set(), set() + self._test_tags = None def tags(self, new_tags, gone_tags): """Handle tag instructions. @@ -260,13 +257,27 @@ class TagCollapsingDecorator(HookedTestResultDecorator): :param new_tags: Tags to add, :param gone_tags: Tags to remove. """ - current_new_tags, current_gone_tags = self._get_current_tags() + current_new_tags, current_gone_tags = self._get_current_scope() current_new_tags.update(new_tags) current_new_tags.difference_update(gone_tags) current_gone_tags.update(gone_tags) current_gone_tags.difference_update(new_tags) +class TagCollapsingDecorator(HookedTestResultDecorator, TagsMixin): + """Collapses many 'tags' calls into one where possible.""" + + def __init__(self, result): + super(TagCollapsingDecorator, self).__init__(result) + self._clear_tags() + + def _before_event(self): + self._flush_current_scope(self.decorated) + + def tags(self, new_tags, gone_tags): + TagsMixin.tags(self, new_tags, gone_tags) + + class TimeCollapsingDecorator(HookedTestResultDecorator): """Only pass on the first and last of a consecutive sequence of times.""" -- cgit v1.2.1 From 16cedbf9742761138fde9be482a118550a69dd1b Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 20 Apr 2012 17:18:26 +0100 Subject: Use the TagsMixin on the predicate so local and global tags are tracked correctly. --- python/subunit/test_results.py | 12 ++++++------ python/subunit/tests/test_subunit_filter.py | 29 ++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 7cce660..59477f9 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -325,14 +325,14 @@ def _make_tag_filter(with_tags, without_tags): return check_tags -class _PredicateFilter(TestResultDecorator): +class _PredicateFilter(TestResultDecorator, TagsMixin): def __init__(self, result, predicate): super(_PredicateFilter, self).__init__(result) + self._clear_tags() self.decorated = TimeCollapsingDecorator( TagCollapsingDecorator(self.decorated)) self._predicate = predicate - self._current_tags = set() # The current test (for filtering tags) self._current_test = None # Has the current test been filtered (for outputting test tags) @@ -344,7 +344,7 @@ class _PredicateFilter(TestResultDecorator): # XXX: ExtendedToOriginalDecorator doesn't properly wrap current_tags. # https://bugs.launchpad.net/testtools/+bug/978027 return self._predicate( - test, outcome, error, details, self._current_tags) + test, outcome, error, details, self._get_active_tags()) def addError(self, test, err=None, details=None): if (self.filter_predicate(test, 'error', err, details)): @@ -394,6 +394,7 @@ class _PredicateFilter(TestResultDecorator): Not directly passed to the client, but used for handling of tags correctly. """ + TagsMixin.startTest(self, test) self._current_test = test self._current_test_filtered = False self._buffered_calls.append(('startTest', [test], {})) @@ -411,11 +412,10 @@ class _PredicateFilter(TestResultDecorator): self._current_test = None self._current_test_filtered = None self._buffered_calls = [] + TagsMixin.stopTest(self, test) def tags(self, new_tags, gone_tags): - new_tags, gone_tags = set(new_tags), set(gone_tags) - self._current_tags.update(new_tags) - self._current_tags.difference_update(gone_tags) + TagsMixin.tags(self, new_tags, gone_tags) if self._current_test is not None: self._buffered_calls.append(('tags', [new_tags, gone_tags], {})) else: diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 7eca4cd..d6da8ce 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -86,12 +86,31 @@ xfail todo result_filter = TestResultFilter( result, filter_success=False, filter_predicate=tag_filter) self.run_tests(result_filter) - test = subunit.RemotedTestCase('passed') + tests_included = [ + event[1] for event in result._events if event[0] == 'startTest'] + tests_expected = map( + subunit.RemotedTestCase, + ['passed', 'error', 'skipped', 'todo']) + self.assertEquals(tests_expected, tests_included) + + def test_tags_tracked_correctly(self): + tag_filter = _make_tag_filter(['a'], []) + result = ExtendedTestResult() + result_filter = TestResultFilter( + result, filter_success=False, filter_predicate=tag_filter) + input_stream = ( + "test: foo\n" + "tags: a\n" + "successful: foo\n" + "test: bar\n" + "successful: bar\n") + self.run_tests(result_filter, input_stream) + foo = subunit.RemotedTestCase('foo') self.assertEquals( - [('tags', set(['global']), set()), - ('startTest', test), - ('addSuccess', test), - ('stopTest', test), + [('startTest', foo), + ('tags', set(['a']), set()), + ('addSuccess', foo), + ('stopTest', foo), ], result._events) -- cgit v1.2.1 From f42ef2fb24def4c19433e18db492416983c56e32 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Thu, 26 Apr 2012 11:50:58 +0100 Subject: Fix up some XXX comments. --- python/subunit/test_results.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'python/subunit') diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 59477f9..5b23b7e 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -341,8 +341,6 @@ class _PredicateFilter(TestResultDecorator, TagsMixin): self._buffered_calls = [] def filter_predicate(self, test, outcome, error, details): - # XXX: ExtendedToOriginalDecorator doesn't properly wrap current_tags. - # https://bugs.launchpad.net/testtools/+bug/978027 return self._predicate( test, outcome, error, details, self._get_active_tags()) @@ -583,7 +581,8 @@ class TestIdPrintingResult(testtools.TestResult): class TestByTestResult(testtools.TestResult): """Call something every time a test completes.""" -# XXX: Arguably belongs in testtools. +# XXX: In testtools since lp:testtools r249. Once that's released, just +# import that. def __init__(self, on_test): """Construct a ``TestByTestResult``. -- cgit v1.2.1