path: root/numpy/testing/parametric.py

"""Support for parametric tests in unittest.

:Author: Fernando Perez

Purpose
=======

Briefly, the main class in this module allows you to easily and cleanly
(without the gross name-mangling hacks that are normally needed) to write
unittest TestCase classes that have parametrized tests.  That is, tests which
consist of multiple sub-tests that scan for example a parameter range, but
where you want each sub-test to:

* count as a separate test in the statistics.

* be run even if others in the group error out or fail.


The class offers a simple name-based convention to create such tests (see
simple example at the end), in one of two ways:

* Each sub-test in a group can be run fully independently, with the
  setUp/tearDown methods being called each time.

* The whole group can be run with setUp/tearDown being called only once for the
  group.  This lets you conveniently reuse state that may be very expensive to
  compute for multiple tests.  Be careful not to corrupt it!!!


Caveats
=======

This code relies on implementation details of the unittest module (some key
methods are heavily modified versions of those, after copying them in).  So it
may well break either if you make sophisticated use of the unittest APIs, or if
unittest itself changes in the future.  I have only tested this with Python
2.5.

"""
__docformat__ = "restructuredtext en"

import unittest

class ParametricTestCase(unittest.TestCase):
    """TestCase subclass with support for parametric tests.

    Subclasses of this class can implement test methods that return a list of
    tests and arguments to call those with, to do parametric testing (often
    also called 'data driven' testing."""

    #: Prefix for tests with independent state.  These methods will be run with
    #: a separate setUp/tearDown call for each test in the group.
    _indepParTestPrefix = 'testip'

    #: Prefix for tests with shared state.  These methods will be run with
    #: a single setUp/tearDown call for the whole group.  This is useful when
    #: writing a group of tests for which the setup is expensive and one wants
    #: to actually share that state.  Use with care (especially be careful not
    #: to mutate the state you are using, which will alter later tests).
    _shareParTestPrefix = 'testsp'

    def exec_test(self,test,args,result):
        """Execute a single test.  Returns a success boolean"""

        ok = False
        try:
            test(*args)
            ok = True
        except self.failureException:
            result.addFailure(self, self._exc_info())
        except KeyboardInterrupt:
            raise
        except:
            result.addError(self, self._exc_info())

        return ok

    def set_testMethodDoc(self,doc):
        self._testMethodDoc = doc
        self._TestCase__testMethodDoc = doc

    def get_testMethodDoc(self):
        return self._testMethodDoc

    testMethodDoc = property(fset=set_testMethodDoc, fget=get_testMethodDoc)

    def get_testMethodName(self):
        try:
            return getattr(self,"_testMethodName")
        except:
            return getattr(self,"_TestCase__testMethodName")

    testMethodName = property(fget=get_testMethodName)

    def run_test(self, testInfo,result):
        """Run one test with arguments"""

        test,args = testInfo[0],testInfo[1:]

        # Reset the doc attribute to be the docstring of this particular test,
        # so that in error messages it prints the actual test's docstring and
        # not that of the test factory.
        self.testMethodDoc = test.__doc__
        result.startTest(self)
        try:
            try:
                self.setUp()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, self._exc_info())
                return

            ok = self.exec_test(test,args,result)

            try:
                self.tearDown()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, self._exc_info())
                ok = False
            if ok: result.addSuccess(self)
        finally:
            result.stopTest(self)

    def run_tests(self, tests,result):
        """Run many tests with a common setUp/tearDown.

        The entire set of tests is run with a single setUp/tearDown call."""

        try:
            self.setUp()
        except KeyboardInterrupt:
            raise
        except:
            result.testsRun += 1
            result.addError(self, self._exc_info())
            return

        saved_doc = self.testMethodDoc

        try:
            # Run all the tests specified
            for testInfo in tests:
                test,args = testInfo[0],testInfo[1:]

                # Set the doc argument for this test.  Note that even if we do
                # this, the fail/error tracebacks still print the docstring for
                # the parent factory, because they only generate the message at
                # the end of the run, AFTER we've restored it.  There is no way
                # to tell the unittest system (without overriding a lot of
                # stuff) to extract this information right away, the logic is
                # hardcoded to pull it later, since unittest assumes it doesn't
                # change.
                self.testMethodDoc = test.__doc__
                result.startTest(self)
                ok = self.exec_test(test,args,result)
                if ok: result.addSuccess(self)

        finally:
            # Restore docstring info and run tearDown once only.
            self.testMethodDoc = saved_doc
            try:
                self.tearDown()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, self._exc_info())

    def run(self, result=None):
        """Test runner."""

        #print
        #print '*** run for method:',self._testMethodName  # dbg
        #print '***            doc:',self._testMethodDoc  # dbg

        if result is None: result = self.defaultTestResult()

        # Independent tests: each gets its own setup/teardown
        if self.testMethodName.startswith(self._indepParTestPrefix):
            for t in getattr(self,self.testMethodName)():
                self.run_test(t,result)
        # Shared-state test: single setup/teardown for all
        elif self.testMethodName.startswith(self._shareParTestPrefix):
            tests = getattr(self,self.testMethodName,'runTest')()
            self.run_tests(tests,result)
        # Normal unittest Test methods
        else:
            unittest.TestCase.run(self,result)

#############################################################################
# Quick and dirty interactive example/test
if __name__ == '__main__':

    class ExampleTestCase(ParametricTestCase):

        #-------------------------------------------------------------------
        # An instrumented setUp method so we can see when it gets called and
        # how many times per instance
        counter = 0

        def setUp(self):
            self.counter += 1
            print 'setUp count: %2s for: %s' % (self.counter,
                                                self.testMethodDoc)

        #-------------------------------------------------------------------
        # A standard test method, just like in the unittest docs.
        def test_foo(self):
            """Normal test for feature foo."""
            pass

        #-------------------------------------------------------------------
        # Testing methods that need parameters.  These can NOT be named test*,
        # since they would be picked up by unittest and called without
        # arguments.  Instead, call them anything else (I use tst*) and then
        # load them via the factories below.
        def tstX(self,i):
            "Test feature X with parameters."
            print 'tstX, i=',i
            if i==1 or i==3:
                # Test fails
                self.fail('i is bad, bad: %s' % i)

        def tstY(self,i):
            "Test feature Y with parameters."
            print 'tstY, i=',i
            if i==1:
                # Force an error
                1/0

        def tstXX(self,i,j):
            "Test feature XX with parameters."
            print 'tstXX, i=',i,'j=',j
            if i==1:
                # Test fails
                self.fail('i is bad, bad: %s' % i)

        def tstYY(self,i):
            "Test feature YY with parameters."
            print 'tstYY, i=',i
            if i==2:
                # Force an error
                1/0

        def tstZZ(self):
            """Test feature ZZ without parameters, needs multiple runs.

            This could be a random test that you want to run multiple times."""
            pass

        #-------------------------------------------------------------------
        # Parametric test factories that create the test groups to call the
        # above tst* methods with their required arguments.
        def testip(self):
            """Independent parametric test factory.

            A separate setUp() call is made for each test returned by this
            method.

            You must return an iterable (list or generator is fine) containing
            tuples with the actual method to be called as the first argument,
            and the arguments for that call later."""
            return [(self.tstX,i) for i in range(5)]

        def testip2(self):
            """Another independent parametric test factory"""
            return [(self.tstY,i) for i in range(5)]

        def testip3(self):
            """Test factory combining different subtests.

            This one shows how to assemble calls to different tests."""
            return [(self.tstX,3),(self.tstX,9),(self.tstXX,4,10),
                    (self.tstZZ,),(self.tstZZ,)]

        def testsp(self):
            """Shared parametric test factory

            A single setUp() call is made for all the tests returned by this
            method.
            """
            return [(self.tstXX,i,i+1) for i in range(5)]

        def testsp2(self):
            """Another shared parametric test factory"""
            return [(self.tstYY,i) for i in range(5)]

        def testsp3(self):
            """Another shared parametric test factory.

            This one simply calls the same test multiple times, without any
            arguments.  Note that you must still return tuples, even if there
            are no arguments."""
            return [(self.tstZZ,) for i in range(10)]


    # This test class runs normally under unittest's default runner
    unittest.main()