Test

Helper functions to quickly write tests in notebooks

All the helpers here wrap test(a,b,cmp): an assert cmp(a,b) that shows both values when the comparison fails, rather than a bare AssertionError. The named comparisons are test_eq, test_ne, test_eq_type (equal and same type), test_close (within eps, default 1e-5), test_is, test_shuffled (equal ignoring order), test_stdout (what f prints), and test_warns.

Simple test functions

To check that code fails as expected, use test_fail(f, contains=..., exc=...) for a callable, or expect_fail as a context manager, which also accepts regex:

---

source

test_fail

def test_fail(
    f, msg:str='', contains:str='', exc:type=Exception, args:NoneType=None, kwargs:NoneType=None
):

Fails with msg unless f() raises an exception of type exc and (optionally) has contains in e.args; expect_fail is normally preferred

def _fail(): raise Exception("foobar")
test_fail(_fail, contains="foo")

def _fail(): raise Exception()
test_fail(_fail)

def _fail(): raise ValueError()
test_fail(_fail, exc=ValueError)
test_fail(lambda: test_fail(_fail, exc=IndexError), exc=AssertionError)

We can also pass args and kwargs to function to check if it fails with special inputs.

def _fail_args(a):
    if a == 5: raise ValueError
test_fail(_fail_args, args=(5,))
test_fail(_fail_args, kwargs=dict(a=5))

---

source

expect_fail

def expect_fail(
    exc:type=Exception, contains:str='', msg:str='', regex:str=''
):

Context manager that fails with msg unless body raises exc, optionally containing contains or matching regex

expect_fail is similar to test_fail, but is a context manager:

with expect_fail(contains="foo"): raise Exception("foobar")
with expect_fail(ValueError): raise ValueError()

# `msg` is included when the check fails: no exception raised, or `contains` not found
with expect_fail(AssertionError, 'no boom'):
    with expect_fail(msg='no boom'): 1+1
with expect_fail(AssertionError, 'wrong text'):
    with expect_fail(contains='foo', msg='wrong text'): raise Exception('bar')

# `regex` is like `contains`, but a regular expression
with expect_fail(regex=r'fo+bar'): raise Exception('foobar')
with expect_fail(AssertionError, 'wrong pat'):
    with expect_fail(regex=r'^bar', msg='wrong pat'): raise Exception('foobar')

def _fail(): raise ValueError('oops')
test_fail(_fail, contains='oops')
with expect_fail(ValueError, regex='o+ps'): _fail()

---

source

test

def test(
    a, b, cmp, cname:NoneType=None
):

assert that cmp(a,b); display inputs and cname or cmp.__name__ if it fails

test([1,2],[1,2], operator.eq)
test_fail(lambda: test([1,2],[1], operator.eq))
test([1,2],[1],   operator.ne)
test_fail(lambda: test([1,2],[1,2], operator.ne))

cmp can be any callable, and fastcore’s curried operators (in_, gt, is_, and friends from fastcore.basics) make good ones: test(x, valid, in_) beats a bare assert x in valid by showing both values when it fails:

from fastcore.basics import in_

test(2, [1,2,3], in_)
test('b', 'abc', in_)

---

all_equal

def all_equal(
    a, b
):

Compares whether a and b are the same length and have the same contents

test(['abc'], ['abc'], all_equal)
test_fail(lambda: test(['abc'],['cab'], all_equal))

---

equals

def equals(
    a, b
):

Compares a and b for equality; supports sublists, tensors and arrays too

test([['abc'],['a']], [['abc'],['a']],  equals)
test([['abc'],['a'],'b', [['x']]], [['abc'],['a'],'b', [['x']]],  equals) # supports any depth and nested structure

---

source

nequals

def nequals(
    a, b
):

Compares a and b for not equals

test(['abc'], ['ab' ], nequals)

test_eq test_ne, etc…

Just use test_eq/test_ne to test for ==/!=. test_eq_type checks things are equal and of the same type. We define them using test:

---

source

test_eq

def test_eq(
    a, b
):

test that a==b

test_eq([1,2],[1,2])
test_eq([1,2],map(int,[1,2]))
test_eq(array([1,2]),array([1,2]))
test_eq([array([1,2]),3],[array([1,2]),3])
test_eq(dict(a=1,b=2), dict(b=2,a=1))
test_fail(lambda: test_eq([1,2], 1), contains="==")
test_fail(lambda: test_eq(None, np.array([1,2])), contains="==")
test_eq({'a', 'b', 'c'}, {'c', 'a', 'b'})

test_eq compares with equals, which works by content across types: generators are consumed, and lists, tuples, sets, dicts, numpy arrays, torch tensors, and DataFrames all compare their contents, including mixed cases like an array against a plain list:

test_eq([1,2], map(int,[1,2]))
test_eq(array([1,2]), [1,2])
test_eq(dict(a=1,b=2), dict(b=2,a=1))

df1 = pd.DataFrame(dict(a=[1,2],b=['a','b']))
df2 = pd.DataFrame(dict(a=[1,2],b=['a','b']))
df3 = pd.DataFrame(dict(a=[1,2],b=['a','c']))

test_eq(df1,df2)
test_eq(df1.a,df2.a)
test_fail(lambda: test_eq(df1,df3), contains='==')
class T(pd.Series): pass
test_eq(df1.iloc[0], T(df2.iloc[0])) # works with subclasses

test_eq(torch.zeros(10), torch.zeros(10, dtype=torch.float64))
test_eq(torch.zeros(10), torch.ones(10)-1)
test_fail(lambda:test_eq(torch.zeros(10), torch.ones(1, 10)), contains='==')
test_eq(torch.zeros(3), [0,0,0])

---

source

test_eq_type

def test_eq_type(
    a, b
):

test that a==b and are same type

test_eq_type(1,1)
test_fail(lambda: test_eq_type(1,1.))
test_eq_type([1,1],[1,1])
test_fail(lambda: test_eq_type([1,1],(1,1)))
test_fail(lambda: test_eq_type([1,1],[1,1.]))

---

source

test_ne

def test_ne(
    a, b
):

test that a!=b

test_ne([1,2],[1])
test_ne([1,2],[1,3])
test_ne(array([1,2]),array([1,1]))
test_ne(array([1,2]),array([1,1]))
test_ne([array([1,2]),3],[array([1,2])])
test_ne([3,4],array([3]))
test_ne([3,4],array([3,5]))
test_ne(dict(a=1,b=2), ['a', 'b'])
test_ne(['a', 'b'], dict(a=1,b=2))

---

source

is_close

def is_close(
    a, b, eps:float=1e-05
):

Is a within eps of b

---

source

test_close

def test_close(
    a, b, eps:float=1e-05
):

test that a is within eps of b

test_close(1,1.001,eps=1e-2)
test_fail(lambda: test_close(1,1.001))
test_close([-0.001,1.001], [0.,1.], eps=1e-2)
test_close(np.array([-0.001,1.001]), np.array([0.,1.]), eps=1e-2)
test_close(array([-0.001,1.001]), array([0.,1.]), eps=1e-2)
test_close(['a','b'], ['a','b'])
test_fail(lambda: test_close(['a','b'], ['a','c']))

---

source

test_is

def test_is(
    a, b
):

test that a is b

test_fail(lambda: test_is([1], [1]))
a = [1]
test_is(a, a)
b = [2]; test_fail(lambda: test_is(a, b))

---

source

test_shuffled

def test_shuffled(
    a, b
):

test that a and b are shuffled versions of the same sequence of items

a = list(range(50))
b = copy(a)
random.shuffle(b)
test_shuffled(a,b)
test_fail(lambda:test_shuffled(a,a))

a = 'abc'
b = 'abcabc'
test_fail(lambda:test_shuffled(a,b))

a = ['a', 42, True] 
b = [42, True, 'a']
test_shuffled(a,b)

---

source

test_stdout

def test_stdout(
    f, exp, regex:bool=False
):

Test that f prints exp to stdout, optionally checking as regex

test_stdout(lambda: print('hi'), 'hi')
test_fail(lambda: test_stdout(lambda: print('hi'), 'ho'))
test_stdout(lambda: 1+1, '')
test_stdout(lambda: print('hi there!'), r'^hi.*!$', regex=True)

---

source

test_warns

def test_warns(
    f, show:bool=False
):

Call self as a function.

test_warns(lambda: warnings.warn("Oh no!"))
test_fail(lambda: test_warns(lambda: 2+2), contains='No warnings raised')

test_warns(lambda: warnings.warn("Oh no!"), show=True)

<class 'UserWarning'>: Oh no!

im = Image.open(TEST_IMAGE).resize((128,128)); im

im = Image.open(TEST_IMAGE_BW).resize((128,128)); im

---

source

test_fig_exists

def test_fig_exists(
    ax
):

Test there is a figure displayed in ax

fig,ax = plt.subplots()
ax.imshow(array(im));

test_fig_exists(ax)

---

source

ExceptionExpected

def ExceptionExpected(
    ex:type=Exception, regex:str=''
):

Context manager that tests if an exception is raised. Deprecated: use expect_fail instead

def _tst_1(): assert False, "This is a test"
def _tst_2(): raise SyntaxError

with ExceptionExpected(): _tst_1()
with ExceptionExpected(ex=AssertionError, regex="This is a test"): _tst_1()
with ExceptionExpected(ex=SyntaxError): _tst_2()

exception is an abbreviation for ExceptionExpected().

with exception: _tst_1()