diff options
| author | Jonathan Lange <jml@canonical.com> | 2011-02-12 11:15:11 +0000 |
|---|---|---|
| committer | Jonathan Lange <jml@canonical.com> | 2011-02-12 11:15:11 +0000 |
| commit | aa2d6df69c425888524124575dd4c5d59e6e8eb8 (patch) | |
| tree | 1fe226c11b12f50b13f2b313b4f28485a2092a83 /python | |
| parent | 148bfa2196c4c6c121c12ff1ae1b7b60a5d5910e (diff) | |
| parent | ab3c87e881b8e9d928aa1845aba52198e7e5f10e (diff) | |
| download | subunit-git-aa2d6df69c425888524124575dd4c5d59e6e8eb8.tar.gz | |
Preserve relative ordering of 'time:' statements (fixes #716554)
Diffstat (limited to 'python')
| -rw-r--r-- | python/subunit/__init__.py | 48 | ||||
| -rw-r--r-- | python/subunit/test_results.py | 37 | ||||
| -rw-r--r-- | python/subunit/tests/test_subunit_filter.py | 173 | ||||
| -rw-r--r-- | python/subunit/tests/test_test_protocol.py | 21 | ||||
| -rw-r--r-- | python/subunit/tests/test_test_results.py | 20 |
5 files changed, 161 insertions, 138 deletions
diff --git a/python/subunit/__init__.py b/python/subunit/__init__.py index b2c7a29..9dc849a 100644 --- a/python/subunit/__init__.py +++ b/python/subunit/__init__.py @@ -6,7 +6,7 @@ # license at the users choice. A copy of both licenses are available in the # project source as Apache-2.0 and BSD. You may not use this file except in # compliance with one of these two licences. -# +# # Unless required by applicable law or agreed to in writing, software # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -49,7 +49,7 @@ details, tags, timestamping and progress markers). The test outcome methods ``addSuccess``, ``addError``, ``addExpectedFailure``, ``addFailure``, ``addSkip`` take an optional keyword parameter ``details`` which can be used instead of the usual python unittest parameter. -When used the value of details should be a dict from ``string`` to +When used the value of details should be a dict from ``string`` to ``testtools.content.Content`` objects. This is a draft API being worked on with the Python Testing In Python mail list, with the goal of permitting a common way to provide additional data beyond a traceback, such as captured data from @@ -58,7 +58,7 @@ and newer). The ``tags(new_tags, gone_tags)`` method is called (if present) to add or remove tags in the test run that is currently executing. If called when no -test is in progress (that is, if called outside of the ``startTest``, +test is in progress (that is, if called outside of the ``startTest``, ``stopTest`` pair), the the tags apply to all sebsequent tests. If called when a test is in progress, then the tags only apply to that test. @@ -87,7 +87,7 @@ tests, allowing isolation between the test runner and some tests. Similarly, ``IsolatedTestCase`` is a base class which can be subclassed to get tests that will fork() before that individual test is run. -`ExecTestCase`` is a convenience wrapper for running an external +`ExecTestCase`` is a convenience wrapper for running an external program to get a Subunit stream and then report that back to an arbitrary result object:: @@ -98,7 +98,7 @@ result object:: def test_script_two(self): './bin/script_two' - + # Normally your normal test loading would take of this automatically, # It is only spelt out in detail here for clarity. suite = unittest.TestSuite([AggregateTests("test_script_one"), @@ -116,7 +116,6 @@ Utility modules * subunit.test_results contains TestResult helper classes. """ -import datetime import os import re from StringIO import StringIO @@ -254,7 +253,7 @@ class _InTest(_ParserState): def _outcome(self, offset, line, no_details, details_state): """An outcome directive has been read. - + :param no_details: Callable to call when no details are presented. :param details_state: The state to switch to for details processing of this outcome. @@ -382,7 +381,7 @@ class _ReadingFailureDetails(_ReadingDetails): def _outcome_label(self): return "failure" - + class _ReadingErrorDetails(_ReadingDetails): """State for the subunit parser when reading error details.""" @@ -430,7 +429,7 @@ class _ReadingSuccessDetails(_ReadingDetails): class TestProtocolServer(object): """A parser for subunit. - + :ivar tags: The current tags associated with the protocol stream. """ @@ -442,7 +441,7 @@ class TestProtocolServer(object): subunit protocol should be written to. This allows custom handling of mixed protocols. By default, sys.stdout will be used for convenience. - :param forward_stream: A stream to forward subunit lines to. This + :param forward_stream: A stream to forward subunit lines to. This allows a filter to forward the entire stream while still parsing and acting on it. By default forward_stream is set to DiscardStream() and no forwarding happens. @@ -510,7 +509,7 @@ class TestProtocolServer(object): def readFrom(self, pipe): """Blocking convenience API to parse an entire stream. - + :param pipe: A file-like object supporting readlines(). :return: None. """ @@ -531,7 +530,7 @@ class TestProtocolServer(object): class TestProtocolClient(testresult.TestResult): """A TestResult which generates a subunit stream for a test run. - + # Get a TestSuite or TestCase to run suite = make_suite() # Create a stream (any object with a 'write' method) @@ -554,7 +553,7 @@ class TestProtocolClient(testresult.TestResult): def addError(self, test, error=None, details=None): """Report an error in test test. - + Only one of error and details should be provided: conceptually there are two separate methods: addError(self, test, error) @@ -569,7 +568,7 @@ class TestProtocolClient(testresult.TestResult): def addExpectedFailure(self, test, error=None, details=None): """Report an expected failure in test test. - + Only one of error and details should be provided: conceptually there are two separate methods: addError(self, test, error) @@ -584,7 +583,7 @@ class TestProtocolClient(testresult.TestResult): def addFailure(self, test, error=None, details=None): """Report a failure in test test. - + Only one of error and details should be provided: conceptually there are two separate methods: addFailure(self, test, error) @@ -599,7 +598,7 @@ class TestProtocolClient(testresult.TestResult): def _addOutcome(self, outcome, test, error=None, details=None): """Report a failure in test test. - + Only one of error and details should be provided: conceptually there are two separate methods: addOutcome(self, test, error) @@ -717,7 +716,7 @@ def RemoteError(description=u""): class RemotedTestCase(unittest.TestCase): """A class to represent test cases run in child processes. - + Instances of this class are used to provide the Python test API a TestCase that can be printed to the screen, introspected for metadata and so on. However, as they are a simply a memoisation of a test that was actually @@ -802,7 +801,7 @@ class ExecTestCase(unittest.TestCase): class IsolatedTestCase(unittest.TestCase): """A TestCase which executes in a forked process. - + Each test gets its own process, which has a performance overhead but will provide excellent isolation from global state (such as django configs, zope utilities and so on). @@ -815,7 +814,7 @@ class IsolatedTestCase(unittest.TestCase): class IsolatedTestSuite(unittest.TestSuite): """A TestSuite which runs its tests in a forked process. - + This decorator that will fork() before running the tests and report the results from the child process using a Subunit stream. This is useful for handling tests that mutate global state, or are testing C extensions that @@ -867,7 +866,7 @@ def run_isolated(klass, self, result): def TAP2SubUnit(tap, subunit): """Filter a TAP pipe into a subunit pipe. - + :param tap: A tap pipe/stream/file object. :param subunit: A pipe/stream/file object to write subunit results to. :return: The exit code to exit with. @@ -875,7 +874,6 @@ def TAP2SubUnit(tap, subunit): BEFORE_PLAN = 0 AFTER_PLAN = 1 SKIP_STREAM = 2 - client = TestProtocolClient(subunit) state = BEFORE_PLAN plan_start = 1 plan_stop = 0 @@ -1025,11 +1023,11 @@ class ProtocolTestCase(object): that has been encoded into the stream. The ``unittest.TestCase`` ``debug`` and ``countTestCases`` methods are not supported because there isn't a sensible mapping for those methods. - + # Get a stream (any object with a readline() method), in this case the # stream output by the example from ``subunit.TestProtocolClient``. stream = file('tests.log', 'rb') - # Create a parser which will read from the stream and emit + # Create a parser which will read from the stream and emit # activity to a unittest.TestResult when run() is called. suite = subunit.ProtocolTestCase(stream) # Create a result object to accept the contents of that stream. @@ -1073,7 +1071,7 @@ class ProtocolTestCase(object): class TestResultStats(testresult.TestResult): """A pyunit TestResult interface implementation for making statistics. - + :ivar total_tests: The total tests seen. :ivar passed_tests: The tests that passed. :ivar failed_tests: The tests that failed. @@ -1124,7 +1122,7 @@ class TestResultStats(testresult.TestResult): def get_default_formatter(): """Obtain the default formatter to write to. - + :return: A file-like object. """ formatter = os.getenv("SUBUNIT_FORMATTER") diff --git a/python/subunit/test_results.py b/python/subunit/test_results.py index 1c91daa..a6ad0c6 100644 --- a/python/subunit/test_results.py +++ b/python/subunit/test_results.py @@ -236,50 +236,51 @@ class TestResultFilter(TestResultDecorator): self._current_test_filtered = None # The (new, gone) tags for the current test. self._current_test_tags = None + # Calls to this result that we don't know whether to forward on yet. + self._buffered_calls = [] def addError(self, test, err=None, details=None): if (not self._filter_error and self.filter_predicate(test, 'error', err, details)): - self.decorated.startTest(test) - self.decorated.addError(test, err, details=details) + self._buffered_calls.append( + ('addError', [test, err], {'details': details})) else: self._filtered() def addFailure(self, test, err=None, details=None): if (not self._filter_failure and self.filter_predicate(test, 'failure', err, details)): - self.decorated.startTest(test) - self.decorated.addFailure(test, err, details=details) + self._buffered_calls.append( + ('addFailure', [test, err], {'details': details})) else: self._filtered() def addSkip(self, test, reason=None, details=None): if (not self._filter_skip and self.filter_predicate(test, 'skip', reason, details)): - self.decorated.startTest(test) - self.decorated.addSkip(test, reason, details=details) + self._buffered_calls.append( + ('addSkip', [reason], {'details': details})) else: self._filtered() def addSuccess(self, test, details=None): if (not self._filter_success and self.filter_predicate(test, 'success', None, details)): - self.decorated.startTest(test) - self.decorated.addSuccess(test, details=details) + self._buffered_calls.append( + ('addSuccess', [test], {'details': details})) else: self._filtered() def addExpectedFailure(self, test, err=None, details=None): if self.filter_predicate(test, 'expectedfailure', err, details): - self.decorated.startTest(test) - return self.decorated.addExpectedFailure(test, err, - details=details) + self._buffered_calls.append( + ('addExpectedFailure', [test, err], {'details': details})) else: self._filtered() def addUnexpectedSuccess(self, test, details=None): - self.decorated.startTest(test) - return self.decorated.addUnexpectedSuccess(test, details=details) + self._buffered_calls.append( + ('addUnexpectedSuccess', [test], {'details': details})) def _filtered(self): self._current_test_filtered = True @@ -293,6 +294,7 @@ class TestResultFilter(TestResultDecorator): self._current_test = test self._current_test_filtered = False self._current_test_tags = set(), set() + self._buffered_calls.append(('startTest', [test], {})) def stopTest(self, test): """Stop a test. @@ -302,12 +304,15 @@ class TestResultFilter(TestResultDecorator): """ if not self._current_test_filtered: # Tags to output for this test. + for method, args, kwargs in self._buffered_calls: + getattr(self.decorated, method)(*args, **kwargs) if self._current_test_tags[0] or self._current_test_tags[1]: self.decorated.tags(*self._current_test_tags) self.decorated.stopTest(test) self._current_test = None self._current_test_filtered = None self._current_test_tags = None + self._buffered_calls = [] def tags(self, new_tags, gone_tags): """Handle tag instructions. @@ -326,6 +331,12 @@ class TestResultFilter(TestResultDecorator): self._current_test_tags[1].difference_update(new_tags) return self.decorated.tags(new_tags, gone_tags) + def time(self, a_time): + if self._current_test is not None: + self._buffered_calls.append(('time', [a_time], {})) + else: + return self.decorated.time(a_time) + def id_to_orig_id(self, id): if id.startswith("subunit.RemotedTestCase."): return id[len("subunit.RemotedTestCase."):] diff --git a/python/subunit/tests/test_subunit_filter.py b/python/subunit/tests/test_subunit_filter.py index 3c65ed3..cf6c2b6 100644 --- a/python/subunit/tests/test_subunit_filter.py +++ b/python/subunit/tests/test_subunit_filter.py @@ -6,7 +6,7 @@ # license at the users choice. A copy of both licenses are available in the # project source as Apache-2.0 and BSD. You may not use this file except in # compliance with one of these two licences. -# +# # Unless required by applicable law or agreed to in writing, software # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -16,119 +16,148 @@ """Tests for subunit.TestResultFilter.""" +from datetime import datetime +from subunit import iso8601 import unittest from StringIO import StringIO +from testtools import TestCase +from testtools.testresult.doubles import ExtendedTestResult + import subunit from subunit.test_results import TestResultFilter -class TestTestResultFilter(unittest.TestCase): +class TestTestResultFilter(TestCase): """Test for TestResultFilter, a TestResult object which filters tests.""" - def _setUp(self): - self.output = StringIO() + # While TestResultFilter works on python objects, using a subunit stream + # is an easy pithy way of getting a series of test objects to call into + # the TestResult, and as TestResultFilter is intended for use with subunit + # also has the benefit of detecting any interface skew issues. + example_subunit_stream = """\ +tags: global +test passed +success passed +test failed +tags: local +failure failed +test error +error error [ +error details +] +test skipped +skip skipped +test todo +xfail todo +""" + + def run_tests(self, result_filter, input_stream=None): + """Run tests through the given filter. + + :param result_filter: A filtering TestResult object. + :param input_stream: Bytes of subunit stream data. If not provided, + uses TestTestResultFilter.example_subunit_stream. + """ + if input_stream is None: + input_stream = self.example_subunit_stream + test = subunit.ProtocolTestCase(StringIO(input_stream)) + test.run(result_filter) def test_default(self): """The default is to exclude success and include everything else.""" - self.filtered_result = unittest.TestResult() - self.filter = TestResultFilter(self.filtered_result) - self.run_tests() + filtered_result = unittest.TestResult() + result_filter = TestResultFilter(filtered_result) + self.run_tests(result_filter) # skips are seen as success by default python TestResult. self.assertEqual(['error'], - [error[0].id() for error in self.filtered_result.errors]) + [error[0].id() for error in filtered_result.errors]) self.assertEqual(['failed'], [failure[0].id() for failure in - self.filtered_result.failures]) - self.assertEqual(4, self.filtered_result.testsRun) + filtered_result.failures]) + self.assertEqual(4, filtered_result.testsRun) def test_exclude_errors(self): - self.filtered_result = unittest.TestResult() - self.filter = TestResultFilter(self.filtered_result, - filter_error=True) - self.run_tests() + filtered_result = unittest.TestResult() + result_filter = TestResultFilter(filtered_result, filter_error=True) + self.run_tests(result_filter) # skips are seen as errors by default python TestResult. - self.assertEqual([], self.filtered_result.errors) + self.assertEqual([], filtered_result.errors) self.assertEqual(['failed'], [failure[0].id() for failure in - self.filtered_result.failures]) - self.assertEqual(3, self.filtered_result.testsRun) + filtered_result.failures]) + self.assertEqual(3, filtered_result.testsRun) def test_exclude_failure(self): - self.filtered_result = unittest.TestResult() - self.filter = TestResultFilter(self.filtered_result, - filter_failure=True) - self.run_tests() + filtered_result = unittest.TestResult() + result_filter = TestResultFilter(filtered_result, filter_failure=True) + self.run_tests(result_filter) self.assertEqual(['error'], - [error[0].id() for error in self.filtered_result.errors]) + [error[0].id() for error in filtered_result.errors]) self.assertEqual([], [failure[0].id() for failure in - self.filtered_result.failures]) - self.assertEqual(3, self.filtered_result.testsRun) + filtered_result.failures]) + self.assertEqual(3, filtered_result.testsRun) def test_exclude_skips(self): - self.filtered_result = subunit.TestResultStats(None) - self.filter = TestResultFilter(self.filtered_result, - filter_skip=True) - self.run_tests() - self.assertEqual(0, self.filtered_result.skipped_tests) - self.assertEqual(2, self.filtered_result.failed_tests) - self.assertEqual(3, self.filtered_result.testsRun) + filtered_result = subunit.TestResultStats(None) + result_filter = TestResultFilter(filtered_result, filter_skip=True) + self.run_tests(result_filter) + self.assertEqual(0, filtered_result.skipped_tests) + self.assertEqual(2, filtered_result.failed_tests) + self.assertEqual(3, filtered_result.testsRun) def test_include_success(self): - """Success's can be included if requested.""" - self.filtered_result = unittest.TestResult() - self.filter = TestResultFilter(self.filtered_result, + """Successes can be included if requested.""" + filtered_result = unittest.TestResult() + result_filter = TestResultFilter(filtered_result, filter_success=False) - self.run_tests() + self.run_tests(result_filter) self.assertEqual(['error'], - [error[0].id() for error in self.filtered_result.errors]) + [error[0].id() for error in filtered_result.errors]) self.assertEqual(['failed'], [failure[0].id() for failure in - self.filtered_result.failures]) - self.assertEqual(5, self.filtered_result.testsRun) + filtered_result.failures]) + self.assertEqual(5, filtered_result.testsRun) def test_filter_predicate(self): """You can filter by predicate callbacks""" - self.filtered_result = unittest.TestResult() + filtered_result = unittest.TestResult() def filter_cb(test, outcome, err, details): return outcome == 'success' - self.filter = TestResultFilter(self.filtered_result, + result_filter = TestResultFilter(filtered_result, filter_predicate=filter_cb, filter_success=False) - self.run_tests() + self.run_tests(result_filter) # Only success should pass - self.assertEqual(1, self.filtered_result.testsRun) - - def run_tests(self): - self.setUpTestStream() - self.test = subunit.ProtocolTestCase(self.input_stream) - self.test.run(self.filter) - - def setUpTestStream(self): - # While TestResultFilter works on python objects, using a subunit - # stream is an easy pithy way of getting a series of test objects to - # call into the TestResult, and as TestResultFilter is intended for - # use with subunit also has the benefit of detecting any interface - # skew issues. - self.input_stream = StringIO() - self.input_stream.write("""tags: global -test passed -success passed -test failed -tags: local -failure failed -test error -error error [ -error details -] -test skipped -skip skipped -test todo -xfail todo -""") - self.input_stream.seek(0) - + self.assertEqual(1, filtered_result.testsRun) + + def test_time_ordering_preserved(self): + # Passing a subunit stream through TestResultFilter preserves the + # relative ordering of 'time' directives and any other subunit + # directives that are still included. + date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC) + date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC) + date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC) + subunit_stream = '\n'.join([ + "time: %s", + "test: foo", + "time: %s", + "error: foo", + "time: %s", + ""]) % (date_a, date_b, date_c) + result = ExtendedTestResult() + result_filter = TestResultFilter(result) + self.run_tests(result_filter, subunit_stream) + foo = subunit.RemotedTestCase('foo') + self.assertEquals( + [('time', date_a), + ('startTest', foo), + ('time', date_b), + ('addError', foo, {}), + ('stopTest', foo), + ('time', date_c)], result._events) + def test_suite(): loader = subunit.tests.TestUtil.TestLoader() diff --git a/python/subunit/tests/test_test_protocol.py b/python/subunit/tests/test_test_protocol.py index e1287b6..f7bab5c 100644 --- a/python/subunit/tests/test_test_protocol.py +++ b/python/subunit/tests/test_test_protocol.py @@ -6,7 +6,7 @@ # license at the users choice. A copy of both licenses are available in the # project source as Apache-2.0 and BSD. You may not use this file except in # compliance with one of these two licences. -# +# # Unless required by applicable law or agreed to in writing, software # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -18,7 +18,6 @@ import datetime import unittest from StringIO import StringIO import os -import sys from testtools.content import Content, TracebackContent from testtools.content_type import ContentType @@ -61,7 +60,6 @@ class TestProtocolServerForward(unittest.TestCase): pipe = StringIO("test old mcdonald\n" "success old mcdonald\n") protocol.readFrom(pipe) - mcdonald = subunit.RemotedTestCase("old mcdonald") self.assertEqual(client.testsRun, 1) self.assertEqual(pipe.getvalue(), out.getvalue()) @@ -74,7 +72,7 @@ class TestProtocolServerForward(unittest.TestCase): protocol.readFrom(pipe) self.assertEqual(client.testsRun, 0) self.assertEqual("", out.getvalue()) - + class TestTestProtocolServerPipe(unittest.TestCase): @@ -90,7 +88,6 @@ class TestTestProtocolServerPipe(unittest.TestCase): "test an error\n" "error an error\n") protocol.readFrom(pipe) - mcdonald = subunit.RemotedTestCase("old mcdonald") bing = subunit.RemotedTestCase("bing crosby") an_error = subunit.RemotedTestCase("an error") self.assertEqual(client.errors, @@ -311,7 +308,7 @@ class TestTestProtocolServerLostConnection(unittest.TestCase): self.protocol.lineReceived("%s old mcdonald %s" % (outcome, opening)) self.protocol.lostConnection() failure = subunit.RemoteError( - u"lost connection during %s report of test 'old mcdonald'" % + u"lost connection during %s report of test 'old mcdonald'" % outcome) self.assertEqual([ ('startTest', self.test), @@ -681,12 +678,6 @@ class TestTestProtocolServerAddSuccess(unittest.TestCase): ], self.client._events) def test_simple_success(self): - self.simple_success_keyword("failure") - - def test_simple_success_colon(self): - self.simple_success_keyword("failure:") - - def test_simple_success(self): self.simple_success_keyword("successful") def test_simple_success_colon(self): @@ -946,7 +937,7 @@ class TestIsolatedTestCase(unittest.TestCase): def test_construct(self): - test = self.SampleIsolatedTestCase("test_sets_global_state") + self.SampleIsolatedTestCase("test_sets_global_state") def test_run(self): result = unittest.TestResult() @@ -982,7 +973,7 @@ class TestIsolatedTestSuite(unittest.TestCase): def test_construct(self): - suite = subunit.IsolatedTestSuite() + subunit.IsolatedTestSuite() def test_run(self): result = unittest.TestResult() @@ -1117,7 +1108,7 @@ class TestTestProtocolClient(unittest.TestCase): self.assertEqual( self.io.getvalue(), 'skip: %s [\nHas it really?\n]\n' % self.test.id()) - + def test_add_skip_details(self): """Test addSkip on a TestProtocolClient with details.""" details = {'reason':Content( diff --git a/python/subunit/tests/test_test_results.py b/python/subunit/tests/test_test_results.py index fe82c04..a2dad96 100644 --- a/python/subunit/tests/test_test_results.py +++ b/python/subunit/tests/test_test_results.py @@ -6,7 +6,7 @@ # license at the users choice. A copy of both licenses are available in the # project source as Apache-2.0 and BSD. You may not use this file except in # compliance with one of these two licences. -# +# # Unless required by applicable law or agreed to in writing, software # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -16,12 +16,6 @@ import datetime import unittest -from StringIO import StringIO -import os -import sys - -from testtools.content_type import ContentType -from testtools.content import Content import subunit import subunit.iso8601 as iso8601 @@ -82,22 +76,22 @@ class TestHookedTestResultDecorator(unittest.TestCase): def test_startTest(self): self.result.startTest(self) - + def test_startTestRun(self): self.result.startTestRun() - + def test_stopTest(self): self.result.stopTest(self) - + def test_stopTestRun(self): self.result.stopTestRun() def test_addError(self): self.result.addError(self, subunit.RemoteError()) - + def test_addError_details(self): self.result.addError(self, details={}) - + def test_addFailure(self): self.result.addFailure(self, subunit.RemoteError()) @@ -142,7 +136,7 @@ class TestHookedTestResultDecorator(unittest.TestCase): def test_time(self): self.result.time(None) - + class TestAutoTimingTestResultDecorator(unittest.TestCase): |
