Utility functions used in the fastai library
from fastcore.test import *
from nbdev.showdoc import *
from PIL import Image
import torch

Basics

ifnone[source]

ifnone(a, b)

b if a is None else a

Since b if a is None else a is such a common pattern, we wrap it in a function. However, be careful, because python will evaluate both a and b when calling ifnone (which it doesn't do if using the if version directly).

test_eq(ifnone(None,1), 1)
test_eq(ifnone(2   ,1), 2)

maybe_attr[source]

maybe_attr(o, attr)

getattr(o,attr,o)

basic_repr[source]

basic_repr(flds=None)

get_class[source]

get_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds)

Dynamically create a class, optionally inheriting from sup, containing fld_names

_t = get_class('_t', 'a', b=2)
t = _t()
test_eq(t.a, None)
test_eq(t.b, 2)
t = _t(1, b=3)
test_eq(t.a, 1)
test_eq(t.b, 3)
t = _t(1, 3)
test_eq(t.a, 1)
test_eq(t.b, 3)
test_eq(repr(t), '_t(a=1, b=3)')
test_eq(t, pickle.loads(pickle.dumps(t)))

Most often you'll want to call mk_class, since it adds the class to your module. See mk_class for more details and examples of use (which also apply to get_class).

mk_class[source]

mk_class(nm, *fld_names, sup=None, doc=None, funcs=None, mod=None, **flds)

Create a class using get_class and add to the caller's module

Any kwargs will be added as class attributes, and sup is an optional (tuple of) base classes.

mk_class('_t', a=1, sup=GetAttr)
t = _t()
test_eq(t.a, 1)
assert(isinstance(t,GetAttr))

A __init__ is provided that sets attrs for any kwargs, and for any args (matching by position to fields), along with a __repr__ which prints all attrs. The docstring is set to doc. You can pass funcs which will be added as attrs with the function names.

def foo(self): return 1
mk_class('_t', 'a', sup=GetAttr, doc='test doc', funcs=foo)

t = _t(3, b=2)
test_eq(t.a, 3)
test_eq(t.b, 2)
test_eq(t.foo(), 1)
test_eq(t.__doc__, 'test doc')
t
<__main__._t at 0x7ff1fa8349d0>

wrap_class[source]

wrap_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds)

Decorator: makes function a method of a new class nm passing parameters to mk_class

@wrap_class('_t', a=2)
def bar(self,x): return x+1

t = _t()
test_eq(t.a, 2)
test_eq(t.bar(3), 4)

class ignore_exceptions[source]

ignore_exceptions()

Context manager to ignore exceptions

with ignore_exceptions(): unknown_func()

noop[source]

noop(x=None, *args, **kwargs)

Do nothing

noop()
test_eq(noop(1),1)

noops[source]

noops(x=None, *args, **kwargs)

Do nothing (method)

mk_class('_t', foo=noops)
test_eq(_t().foo(1),1)

store_attr[source]

store_attr(nms)

Store params named in comma-separated nms from calling context into attrs in self

class T:
    def __init__(self, a,b,c): store_attr(self, 'a,b, c')

t = T(1,c=2,b=3)
assert t.a==1 and t.b==3 and t.c==2

attrdict[source]

attrdict(o, *ks)

Dict from each k in ks to getattr(o,k)

test_eq(attrdict(t,'b','c'), {'b':3, 'c':2})

properties[source]

properties(*ps)

Change attrs in cls with names in ps to properties

class T:
    def a(self): return 1
    def b(self): return 2
properties(T,'a')

test_eq(T().a,1)
test_eq(T().b(),2)

camel2snake[source]

camel2snake(name)

Convert CamelCase to snake_case

test_eq(camel2snake('ClassAreCamel'), 'class_are_camel')

snake2camel[source]

snake2camel(s)

Convert snake_case to CamelCase

test_eq(snake2camel('a_b_cc'), 'ABCc')

class2attr[source]

class2attr(cls_name)

hasattrs[source]

hasattrs(o, attrs)

Test whether o contains all attrs

assert hasattrs(1,('imag','real'))
assert not hasattrs(1,('imag','foo'))

class ShowPrint[source]

ShowPrint()

Base class that prints for show

class Int[source]

Int() :: int

An extensible int

class Float[source]

Float(x=0) :: float

An extensible float

class Str[source]

Str() :: str

An extensible str

Collection functions

tuplify[source]

tuplify(o, use_list=False, match=None)

Make o a tuple

test_eq(tuplify(None),())
test_eq(tuplify([1,2,3]),(1,2,3))
test_eq(tuplify(1,match=[1,2,3]),(1,1,1))

detuplify[source]

detuplify(x)

If x is a tuple with one thing, extract it

test_eq(detuplify(()),None)
test_eq(detuplify([1]),1)
test_eq(detuplify([1,2]), [1,2])
test_eq(detuplify(np.array([[1,2]])), np.array([[1,2]]))

replicate[source]

replicate(item, match)

Create tuple of item copied len(match) times

t = [1,1]
test_eq(replicate([1,2], t),([1,2],[1,2]))
test_eq(replicate(1, t),(1,1))

uniqueify[source]

uniqueify(x, sort=False, bidir=False, start=None)

Return the unique elements in x, optionally sort-ed, optionally return the reverse correspondence.

test_eq(set(uniqueify([1,1,0,5,0,3])),{0,1,3,5})
test_eq(uniqueify([1,1,0,5,0,3], sort=True),[0,1,3,5])
v,o = uniqueify([1,1,0,5,0,3], bidir=True)
test_eq(v,[1,0,5,3])
test_eq(o,{1:0, 0: 1, 5: 2, 3: 3})
v,o = uniqueify([1,1,0,5,0,3], sort=True, bidir=True)
test_eq(v,[0,1,3,5])
test_eq(o,{0:0, 1: 1, 3: 2, 5: 3})

setify[source]

setify(o)

test_eq(setify(None),set())
test_eq(setify('abc'),{'abc'})
test_eq(setify([1,2,2]),{1,2})
test_eq(setify(range(0,3)),{0,1,2})
test_eq(setify({1,2}),{1,2})

merge[source]

merge(*ds)

Merge all dictionaries in ds

test_eq(merge(), {})
test_eq(merge(dict(a=1,b=2)), dict(a=1,b=2))
test_eq(merge(dict(a=1,b=2), dict(b=3,c=4), None), dict(a=1, b=3, c=4))

is_listy[source]

is_listy(x)

isinstance(x, (tuple,list,L))

assert is_listy([1])
assert is_listy(L([1]))
assert is_listy(slice(2))
assert not is_listy(array([1]))

range_of[source]

range_of(x)

All indices of collection x (i.e. list(range(len(x))))

test_eq(range_of([1,1,1,1]), [0,1,2,3])

groupby[source]

groupby(x, key)

Like itertools.groupby but doesn't need to be sorted, and isn't lazy

test_eq(groupby('aa ab bb'.split(), itemgetter(0)), {'a':['aa','ab'], 'b':['bb']})

first[source]

first(x)

First element of x, or None if missing

shufflish[source]

shufflish(x, pct=0.04)

Randomly relocate items of x up to pct of len(x) from their starting location

l = list(range(100))
l2 = array(shufflish(l))
test_close(l2[:50 ].mean(), 25, eps=5)
test_close(l2[-50:].mean(), 75, eps=5)
test_ne(l,l2)

class IterLen[source]

IterLen()

Base class to add iteration to anything supporting len and __getitem__

class ReindexCollection[source]

ReindexCollection(coll, idxs=None, cache=None, tfm='noop') :: GetAttr

Reindexes collection coll with indices idxs and optional LRU cache of size cache

sz = 50
t = ReindexCollection(L.range(sz), cache=2)
test_eq(list(t), range(sz))
test_eq(t[sz-1], sz-1)
test_eq(t._get.cache_info().hits, 1)
t.shuffle()
test_eq(t._get.cache_info().hits, 1)
test_ne(list(t), range(sz))
test_eq(set(t), set(range(sz)))
t.cache_clear()
test_eq(t._get.cache_info().hits, 0)
test_eq(t.count(0), 1)

in_[source]

in_(a, b=nan)

Same as operator.in_, or returns partial if 1 arg

The following functions are provided matching the behavior of the equivalent versions in operator: lt gt le ge eq ne add sub mul truediv is_ is_not. In addition one method is added to operator, which is in_(x,a), which is the same as x in a.

lt(3,5),gt(3,5),is_(None,None),in_(0,[1,2])
(True, False, True, False)

However, they also have additional functionality: if you only pass one param, they return a partial function that passes that param as the second positional parameter.

lt(5)(3),gt(5)(3),is_(None)(None),in_([1,2])(0)
(True, False, True, False)

class Inf[source]

Inf()

Infinite lists

Inf defines the following properties:

  • count: itertools.count()
  • zeros: itertools.cycle([0])
  • ones : itertools.cycle([1])
  • nones: itertools.cycle([None])
test_eq([o for i,o in zip(range(5), Inf.count)],
        [0, 1, 2, 3, 4])

test_eq([o for i,o in zip(range(5), Inf.zeros)],
        [0, 0, 0, 0, 0])

true[source]

true(*args, **kwargs)

Predicate: always True

stop[source]

stop(e='StopIteration')

Raises exception e (by default StopException) even if in an expression

gen[source]

gen(func, seq, cond='true')

Like (func(o) for o in seq if cond(func(o))) but handles StopIteration

test_eq(gen(noop, Inf.count, lt(5)),
        range(5))
test_eq(gen(operator.neg, Inf.count, gt(-5)),
        [0,-1,-2,-3,-4])
test_eq(gen(lambda o:o if o<5 else stop(), Inf.count),
        range(5))

chunked[source]

chunked(it, cs, drop_last=False)

t = L.range(10)
test_eq(chunked(t,3),      [[0,1,2], [3,4,5], [6,7,8], [9]])
test_eq(chunked(t,3,True), [[0,1,2], [3,4,5], [6,7,8],    ])

t = map(lambda o:stop() if o==6 else o, Inf.count)
test_eq(chunked(t,3), [[0, 1, 2], [3, 4, 5]])
t = map(lambda o:stop() if o==7 else o, Inf.count)
test_eq(chunked(t,3), [[0, 1, 2], [3, 4, 5], [6]])

t = np.arange(10)
test_eq(chunked(t,3),      L([0,1,2], [3,4,5], [6,7,8], [9]))
test_eq(chunked(t,3,True), L([0,1,2], [3,4,5], [6,7,8],    ))

class Tuple[source]

Tuple(x=None, *rest) :: tuple

A tuple with elementwise ops and more friendly init behavior

test_eq(Tuple(1), (1,))
test_eq(type(Tuple(1)), Tuple)
test_eq_type(Tuple(1,2), Tuple(1,2))
test_ne(Tuple(1,2), Tuple(1,3))
test_eq(Tuple(), ())
test_eq(Tuple((1,2)), (1,2))
test_eq(-Tuple(1,2), (-1,-2))
test_eq(Tuple(1,1)-Tuple(2,2), (-1,-1))
test_eq(Tuple.add((1,1),(2,2)), (3,3))
test_eq(Tuple(1,1).add((2,2)), Tuple(3,3))
test_eq(Tuple('1','2').add('2'), Tuple('12','22'))
test_eq_type(Tuple(1,1).add(2), Tuple(3,3))
test_eq_type(Tuple(1,1).mul(2), Tuple(2,2))
test_eq(Tuple(3,1).le(1), (False, True))
test_eq(Tuple(3,1).eq(1), (False, True))
test_eq(Tuple(3,1).gt(1), (True, False))
test_eq(Tuple(3,1).min(2), (2,1))
test_eq(~Tuple(1,0,1), (False,True,False))

Functions on functions

trace[source]

trace(f)

Add set_trace to an existing function f

compose[source]

compose(*funcs, order=None)

Create a function that composes all functions in funcs, passing along remaining *args and **kwargs to all

f1 = lambda o,p=0: (o*2)+p
f2 = lambda o,p=1: (o+1)/p
test_eq(f2(f1(3)), compose(f1,f2)(3))
test_eq(f2(f1(3,p=3),p=3), compose(f1,f2)(3,p=3))
test_eq(f2(f1(3,  3),  3), compose(f1,f2)(3,  3))

f1.order = 1
test_eq(f1(f2(3)), compose(f1,f2, order="order")(3))

maps[source]

maps(*args, retain='noop')

Like map, except funcs are composed first

test_eq(maps([1]), [1])
test_eq(maps(operator.neg, [1,2]), [-1,-2])
test_eq(maps(operator.neg, operator.neg, [1,2]), [1,2])

partialler[source]

partialler(f, *args, order=None, **kwargs)

Like functools.partial but also copies over docstring

def _f(x,a=1):
    "test func"
    return x+a
_f.order=1

f = partialler(_f, a=2)
test_eq(f.order, 1)
f = partialler(_f, a=2, order=3)
test_eq(f.__doc__, "test func")
test_eq(f.order, 3)
test_eq(f(3), _f(3,2))

mapped[source]

mapped(f, it)

map f over it, unless it's not listy, in which case return f(it)

test_eq(mapped(_f,1),2)
test_eq(mapped(_f,[1,2]),[2,3])
test_eq(mapped(_f,(1,)),(2,))

instantiate[source]

instantiate(t)

Instantiate t if it's a type, otherwise do nothing

test_eq_type(instantiate(int), 0)
test_eq_type(instantiate(1), 1)

using_attr[source]

using_attr(f, attr)

Change function f to operate on attr

t = Path('/a/b.txt')
f = using_attr(str.upper, 'name')
test_eq(f(t), 'B.TXT')
def sleep(func=None, *, seconds=None, msg=None):
    if func is None:
        return partial(sleep, seconds=seconds, msg=msg)

    seconds = seconds if seconds else 1
    msg = msg if msg else 'Sleeping for {} seconds'.format(seconds)

    @wraps(func)
    def wrapper(*args, **kwargs):
        print(msg)
        time.sleep(seconds)
        return func(*args, **kwargs)
    return wrapper

log_args[source]

log_args(f=None, to_return=False, but=None, but_as=None)

Decorator to log function args in 'to.init_args'

class tst:
    @log_args
    def __init__(self, a, b, c=3, d=4):
        pass
test_eq(tst(1,2).init_args, {f'tst.__init__.{k}':v for k,v in dict(a=1,b=2,c=3,d=4).items()})

Use log_args to save function args in to.init_args. Optional args are:

  • to_return: applies to return value if True (for functions), otherwise to self (for class instances)
  • but: args that we do not want to save separated by ','
  • but_as: pull but arg from another log_args (which cannot have used to_return=True)

Notes:

  • @log_args needs to be placed below @patch and above @funcs_kwargs or @delegates
  • when wrapping a class, it will wrap its __init__ method
class tst:
    @log_args
    def __init__(self, a, b, c=3, d=4):
        pass
test_eq(tst(1,2).init_args, {f'tst.__init__.{k}':v for k,v in dict(a=1,b=2,c=3,d=4).items()})

@log_args
class tst:
    def __init__(self, a, b, c=3, d=4):
        pass
test_eq(tst(1,2).init_args, {f'tst.__init__.{k}':v for k,v in dict(a=1,b=2,c=3,d=4).items()})

@log_args(but='a,c')
class tst:
    def __init__(self, a, b, c=3, d=4):
        pass
test_eq(tst(1,2).init_args, {f'tst.__init__.{k}':v for k,v in dict(b=2,d=4).items()})

class tst:
    pass
@log_args(to_return=True)
def tst_f(a,b): return tst
test_eq(tst_f(1,2).init_args, {f'tst_f.{k}':v for k,v in dict(a=1,b=2).items()})

@log_args
@funcs_kwargs
class tst:
    _methods='a'.split()    
    def __init__(self, **kwargs):
        pass
test_eq(tst(a=noop).init_args['tst.__init__.a'], noop)

class tst_base:
    def __init__(self, a=None):
        pass
@log_args
@delegates(tst_base)
class tst:
    def __init__(self, **kwargs):
        pass
test_eq(tst(a=1).init_args['tst.__init__.a'], 1)

@log_args
class tst_parent:
    def __init__(self, a):
        pass
@log_args
class tst(tst_parent):
    def __init__(self, b):
        super().__init__(a=b)
test_eq(tst(1).init_args, {'tst_parent.__init__.a': 1, 'tst.__init__.b': 1})

class tst_ref:
    def __init__(self, a, b, c, d):
        pass
class tst:
    def __init__(self, a=1, b=2, c=3, d=4):
        pass
test_stdout(lambda: log_args(tst, but_as=tst_ref.__init__), '@log_args did not find args from but_as while wrapping tst.__init__ in tst_ref.__init__')

@log_args(but='a,b')
class tst_ref:
    def __init__(self, a, b, c, d):
        pass
@log_args(but_as=tst_ref.__init__)
class tst:
    def __init__(self, a=1, b=2, c=3, d=4):
        pass
test_eq(tst().init_args, {'tst.__init__.c': 3, 'tst.__init__.d': 4})

@log_args(but='a,b')
class tst_ref:
    def __init__(self, a, b, c, d):
        pass
@log_args(but='c', but_as=tst_ref.__init__)
class tst:
    def __init__(self, a=1, b=2, c=3, d=4):
        pass
test_eq(tst().init_args, {'tst.__init__.d': 4})

Self

fastai provides a concise way to create lambdas that are calling methods on an object, which is to use Self (note the capitalization!) Self.sum()), for instance, is a shortcut for lambda o: o.sum().

f = Self.sum()
x = array([3.,1])
test_eq(f(x), 4.)

# This is equivalent to above
f = lambda o: o.sum()
x = array([3.,1])
test_eq(f(x), 4.)

f = Self.sum().is_integer()
x = array([3.,1])
test_eq(f(x), True)

f = Self.sum().real.is_integer()
x = array([3.,1])
test_eq(f(x), True)

f = Self.imag()
test_eq(f(3), 0)

f = Self[1]
test_eq(f(x), 1)

File and network functions

Path.readlines[source]

Path.readlines(hint=-1, encoding='utf8')

Read the content of fname

Path.read[source]

Path.read(size=-1, encoding='utf8')

Read the content of fname

Path.write[source]

Path.write(txt, encoding='utf8')

Write txt to self, creating directories as needed

with tempfile.NamedTemporaryFile() as f:
    fn = Path(f.name)
    fn.write('t')
    t = fn.read()
    test_eq(t,'t')
    t = fn.readlines()
    test_eq(t,['t'])

Path.save[source]

Path.save(fn:Path, o)

Save a pickle file, to a file name or opened file

Path.load[source]

Path.load(fn:Path)

Load a pickle file from a file name or opened file

with tempfile.NamedTemporaryFile() as f:
    fn = Path(f.name)
    fn.save('t')
    t = fn.load()
test_eq(t,'t')

Path.ls[source]

Path.ls(n_max=None, file_type=None, file_exts=None)

Contents of path as a list

We add an ls() method to pathlib.Path which is simply defined as list(Path.iterdir()), mainly for convenience in REPL environments such as notebooks.

path = Path()
t = path.ls()
assert len(t)>0
t1 = path.ls(10)
test_eq(len(t1), 10)
t2 = path.ls(file_exts='.ipynb')
assert len(t)>len(t2)
t[0]
PosixPath('00_test.ipynb')

You can also pass an optional file_type MIME prefix and/or a list of file extensions.

lib_path = (path/'../fastcore')
txt_files=lib_path.ls(file_type='text')
assert len(txt_files) > 0 and txt_files[0].suffix=='.py'
ipy_files=path.ls(file_exts=['.ipynb'])
assert len(ipy_files) > 0 and ipy_files[0].suffix=='.ipynb'
txt_files[0],ipy_files[0]
(PosixPath('../fastcore/dispatch.py'), PosixPath('00_test.ipynb'))

Path.__repr__[source]

Path.__repr__()

Return repr(self).

fastai also updates the repr of Path such that, if Path.BASE_PATH is defined, all paths are printed relative to that path (as long as they are contained in Path.BASE_PATH:

t = ipy_files[0].absolute()
try:
    Path.BASE_PATH = t.parent.parent
    test_eq(repr(t), f"Path('nbs/{t.name}')")
finally: Path.BASE_PATH = None

bunzip[source]

bunzip(fn)

bunzip fn, raising exception if output already exists

f = Path('files/test.txt')
if f.exists(): f.unlink()
bunzip('files/test.txt.bz2')
t = f.open().readlines()
test_eq(len(t),1)
test_eq(t[0], 'test\n')
f.unlink()

join_path_file[source]

join_path_file(file, path, ext='')

Return path/file if file is a string or a Path, file otherwise

path = Path.cwd()/'_tmp'/'tst'
f = join_path_file('tst.txt', path)
assert path.exists()
test_eq(f, path/'tst.txt')
with open(f, 'w') as f_: assert join_path_file(f_, path) == f_
shutil.rmtree(Path.cwd()/'_tmp')

remove_patches_path[source]

remove_patches_path()

with remove_patches_path():
    assert not hasattr(Path, 'write')
assert hasattr(Path, 'write')

Sorting objects from before/after

Transforms and callbacks will have run_after/run_before attributes, this function will sort them to respect those requirements (if it's possible). Also, sometimes we want a tranform/callback to be run at the end, but still be able to use run_after/run_before behaviors. For those, the function checks for a toward_end attribute (that needs to be True).

sort_by_run[source]

sort_by_run(fs)

class Tst(): pass    
class Tst1():
    run_before=[Tst]
class Tst2():
    run_before=Tst
    run_after=Tst1
    
tsts = [Tst(), Tst1(), Tst2()]
test_eq(sort_by_run(tsts), [tsts[1], tsts[2], tsts[0]])

Tst2.run_before,Tst2.run_after = Tst1,Tst
test_fail(lambda: sort_by_run([Tst(), Tst1(), Tst2()]))

def tst1(x): return x
tst1.run_before = Tst
test_eq(sort_by_run([tsts[0], tst1]), [tst1, tsts[0]])
    
class Tst1():
    toward_end=True
class Tst2():
    toward_end=True
    run_before=Tst1
tsts = [Tst(), Tst1(), Tst2()]
test_eq(sort_by_run(tsts), [tsts[0], tsts[2], tsts[1]])

Other helpers

class PrettyString[source]

PrettyString() :: str

Little hack to get strings to show properly in Jupyter.

round_multiple[source]

round_multiple(x, mult, round_down=False)

Round x to nearest multiple of mult

test_eq(round_multiple(63,32), 64)
test_eq(round_multiple(50,32), 64)
test_eq(round_multiple(40,32), 32)
test_eq(round_multiple( 0,32),  0)
test_eq(round_multiple(63,32, round_down=True), 32)
test_eq(round_multiple((63,40),32), (64,32))

even_mults[source]

even_mults(start, stop, n)

Build log-stepped array from start to stop in n steps.

test_eq(even_mults(2,8,3), [2,4,8])
test_eq(even_mults(2,32,5), [2,4,8,16,32])
test_eq(even_mults(2,8,1), 8)

num_cpus[source]

num_cpus()

Get number of cpus

add_props[source]

add_props(f, g=None, n=2)

Create properties passing each of range(n) to f

class _T(): a,b = add_props(lambda i,x:i*2)

t = _T()
test_eq(t.a,0)
test_eq(t.b,2)
class _T(): 
    def __init__(self, v): self.v=v
    def _set(i, self, v): self.v[i] = v
    a,b = add_props(lambda i,x: x.v[i], _set)

t = _T([0,2])
test_eq(t.a,0)
test_eq(t.b,2)
t.a = t.a+1
t.b = 3
test_eq(t.a,1)
test_eq(t.b,3)

change_attr[source]

change_attr(o, name, new_val)

Change the attr name in o with new_val if it exists and return the old value

change_attrs[source]

change_attrs(o, names, new_vals, do=None)

Change the attr names in o with new_vals if it exists and return the old values