Providing test data in Python

Question

How can I run the same test against a lot of different data?

I want to be reported of all failures.

For example:

def isEven(number):
    return True # Quite buggy implementation

data = [
    (2, True),
    (3, False),
    (4, True),
    (5, False),
]

class MyTest:
   def evenTest(self, num, expected):
       self.assertEquals(expected, isEven(num))

I have found solution which raises error on first failure only: PHPUnit style dataProvider in Python unit test

How can I run a test to be reported of all failures?

Answer 1

If you are using pytest you can go this way:

import pytest

def is_even(number):
    return True # Wuite buggy implementation

@pytest.mark.parametrize("number, expected", [
    (2, True),
    (3, False),
    (4, True),
    (5, False)
])
def test_is_even(number, expected):
    assert is_even(number) == expected

You will get something like (shortened):

/tmp/test_it.py:13: AssertionError
=========== 2 failed, 2 passed in 0.01 seconds ====================

Answer 2

One solution is to make different test case instances for each entry in data:

class MyTest(unittest.TestCase):
    def __init__(self, num, expected):
        unittest.TestCase.__init__(self, "evenTest")
        self.num = num
        self.expected = expected
    def evenTest(self):
        self.assertEqual(self.expected, isEven(self.num))

For unittest to know how to construct the test cases, add a load_tests() function to your module:

def load_tests(loader, tests, pattern):
    return unittest.TestSuite(MyTest(num, expected)
                              for num, expected in data)

Answer 3

You should be using py.test. I think the unittest module was blindly copied from JUnit. Anyway, you can hack your way like this:

import unittest

data = [
    (2, True),
    (3, False),
    (4, True),
    (5, False)]

# This should be imported from a separate module.
def isEven(number):
    return True # Quite buggy implementation

def create_test_func(num, expected):
    def _test_func(self):
        self.assertEqual(expected, isEven(num))
    return _test_func

class TestIsEven(unittest.TestCase):

    pass

# pyunit isn't Pythonic enough. Use py.test instead
# till then we rely on such hackery
import new
for i, (num, expected) in enumerate(data):
    setattr(TestIsEven, 'test_data_%d'%i, create_test_func(num, expected))

if __name__ == "__main__":
    unittest.main()

And the output is:

.F.F
======================================================================
FAIL: test_data_1 (__main__.TestIsEven)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "untitled-1.py", line 15, in _test_func
    self.assertEqual(expected, isEven(num))
AssertionError: False != True

======================================================================
FAIL: test_data_3 (__main__.TestIsEven)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "untitled-1.py", line 15, in _test_func
    self.assertEqual(expected, isEven(num))
AssertionError: False != True

----------------------------------------------------------------------
Ran 4 tests in 0.000s

FAILED (failures=2)

Using this approach, you can add more niceties like printing debugging information on failure, etc.

Answer 4

You may be looking for something like this:

import unittest


def is_even(number):
    return True # Quite buggy implementation


class TestCase(unittest.TestCase):
    def setUp(self):
        self.expected_output = [
            (2, True),
            (3, False),
            (4, True),
            (5, False)
        ]

    def test_is_even(self):
        real_res = []

        for arg, _ in self.expected_output:
            real_res.append((arg, is_even(arg)))

        msg_error = '\nFor %s Expected %s Got %s'
        msg = []
        for res1, res2 in zip(real_res, self.expected_output):
            if res1[1] != res2[1]:
                msg.append(msg_error % (res1[0], res1[1], res2[1]))


        self.assertEqual(real_res, self.expected_output, "".join(msg))


if __name__ == '__main__':
    unittest.main()

Output:

F
======================================================================
FAIL: test_is_even (__main__.TestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "test.py", line 29, in test_example
    self.assertEqual(real_res, self.expected_output, ''.join(msg))
AssertionError:
For 3 Expected True Got False
For 5 Expected True Got False

----------------------------------------------------------------------
Ran 1 test in 0.000s

FAILED (failures=1)

Answer 5

import unittest

data = [
    (2, True),
    (3, False),
    (4, True),
    (5, False)]

# This should be imported from a separate module.
def isEven(number):
    return True # Quite buggy implementation


class TestIsEven(unittest.TestCase):
    def test_is_even(self):
        for num, expected in data:
            self.assertEqual(expected, isEven(num))