Basic functions used in the fastai library
from fastcore.test import *
from nbdev.showdoc import *

Metaclasses

See this blog post for more information about metaclasses.

  • PrePostInitMeta ensures that the classes defined with it run __pre_init__ and __post_init__ (without having to write self.__pre_init__() and self.__post_init__() in the actual init
  • NewChkMeta gives the PrePostInitMeta functionality and ensures classes defined with it don't re-create an object of their type whenever it's passed to the constructor
  • BypassNewMeta ensures classes defined with it can easily be casted form objects they subclass.

class FixSigMeta[source]

FixSigMeta(name, bases, dict) :: type

A metaclass that fixes the signature on classes that override new

class PrePostInitMeta[source]

PrePostInitMeta(name, bases, dict) :: FixSigMeta

A metaclass that calls optional __pre_init__ and __post_init__ methods

class _T(metaclass=PrePostInitMeta):
    def __pre_init__(self):  self.a  = 0; assert self.a==0
    def __init__(self,b=0):  self.a += 1; assert self.a==1
    def __post_init__(self): self.a += 1; assert self.a==2

t = _T()
test_eq(t.a, 2)

class NewChkMeta[source]

NewChkMeta(name, bases, dict) :: FixSigMeta

Metaclass to avoid recreating object passed to constructor

class _T(metaclass=NewChkMeta):
    "Testing"
    def __init__(self, o=None, b=1):
        self.foo = getattr(o,'foo',0) + 1
        self.b = b
class _T2():
    def __init__(self, o): self.foo = getattr(o,'foo',0) + 1

t = _T(1)
test_eq(t.foo,1)
t2 = _T(t)
test_eq(t2.foo,1)
test_is(t,t2)
t3 = _T(t, b=2)
test_eq(t3.b, 2)
assert not t3 is t

t = _T2(1)
test_eq(t.foo,1)
t2 = _T2(t)
test_eq(t2.foo,2)

test_eq(_T.__doc__, "Testing")
test_eq(str(inspect.signature(_T)), '(o=None, b=1)')

class BypassNewMeta[source]

BypassNewMeta(name, bases, dict) :: FixSigMeta

Metaclass: casts x to this class if it's of type cls._bypass_type, initializing with _new_meta if available

class T0: pass
class _T(T0, metaclass=BypassNewMeta):
    _bypass_type=T0
    def __init__(self,x): self.x=x

t = T0()
t.a = 1
t2 = _T(t)
test_eq(type(t2), _T)
test_eq(t2.a,1)
test_is(t2,t)
t = _T(2)
t.x = 2

Foundational functions

copy_func[source]

copy_func(f)

Copy a non-builtin function (NB copy.copy does not work for this)

patch_to[source]

patch_to(as_prop=False, cls_method=False)

Decorator: add f to cls

class _T3(int): pass

@patch_to(_T3)
def func1(x, a): return x+a

@patch_to(_T3, cls_method=True)
def from_func2(cls, a): return cls(a)

t = _T3(1)
test_eq(t.func1(2), 3)

t2 = _T3(2)
test_eq(t2, 2)

If cls is a tuple, f is added to all types in the tuple.

class _T4(int): pass
@patch_to((_T3,_T4))
def func2(x, a): return x+2*a

t = _T3(1)
test_eq(t.func2(1), 3)
t = _T4(1)
test_eq(t.func2(1), 3)

patch[source]

patch(f)

Decorator: add f to the first parameter's class (based on f's type annotations)

@patch
def func(x:_T3, a):
    "test"
    return x+a

t = _T3(1)
test_eq(t.func(3), 4)
test_eq(t.func.__qualname__, '_T3.func')

If annotation is a tuple, the function is added to all types in the tuple.

@patch
def func3(x:(_T3,_T4), a):
    "test"
    return x+2*a

t = _T3(1)
test_eq(t.func3(2), 5)
test_eq(t.func3.__qualname__, '_T3.func3')
t = _T4(1)
test_eq(t.func3(2), 5)
test_eq(t.func3.__qualname__, '_T4.func3')

patch_property[source]

patch_property(f)

Decorator: add f as a property to the first parameter's class (based on f's type annotations)

@patch_property
def prop(x:_T3): return x+1

t = _T3(1)
test_eq(t.prop, 2)
def test_sig(f, b): test_eq(str(inspect.signature(f)), b)

use_kwargs_dict[source]

use_kwargs_dict(keep=False, **kwargs)

Decorator: replace **kwargs in signature with names params

@use_kwargs_dict(y=1,z=None)
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, *, y=1, z=None)')

use_kwargs[source]

use_kwargs(names, keep=False)

Decorator: replace **kwargs in signature with names params

@use_kwargs(['y', 'z'])
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, *, y=None, z=None)')

@use_kwargs(['y', 'z'], keep=True)
def foo(a, *args, b=1, **kwargs): pass
test_sig(foo, '(a, *args, b=1, y=None, z=None, **kwargs)')

delegates[source]

delegates(to=None, keep=False, but=None)

Decorator: replace **kwargs in signature with params from to

def basefoo(e, c=2): pass

@delegates(basefoo)
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, c=2)')

@delegates(basefoo, keep=True)
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, c=2, **kwargs)')


@delegates(basefoo, but= ['c'])
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1)')

class _T():
    @classmethod
    def foo(cls, a=1, b=2):
        pass
    
    @classmethod
    @delegates(foo)
    def bar(cls, c=3, **kwargs):
        pass

test_sig(_T.bar, '(c=3, a=1, b=2)')
class BaseFoo:
    def __init__(self, e, c=2): pass

@delegates()
class Foo(BaseFoo):
    def __init__(self, a, b=1, **kwargs): super().__init__(**kwargs)

test_sig(Foo, '(a, b=1, c=2)')

funcs_kwargs[source]

funcs_kwargs()

Replace methods in cls._methods with those from kwargs

method[source]

method(f)

Mark f as a method

@funcs_kwargs
class T:
    _methods=['b']
    def __init__(self, f=1, **kwargs): assert not kwargs
    def a(self): return 1
    def b(self): return 2
    
t = T()
test_eq(t.a(), 1)
test_eq(t.b(), 2)
t = T(b = lambda:3)
test_eq(t.b(), 3)
test_sig(T, '(f=1, *, b=None)')
test_fail(lambda: T(a = lambda:3))

@method
def _f(self,a=1): return a+1
t = T(b = _f)
test_eq(t.b(2), 3)

class T2(T):
    def __init__(self,a):
        super().__init__(b = lambda:3)
        self.a=a
t = T2(a=1)
test_eq(t.b(), 3)
test_sig(T2, '(a)')

def _g(a=1): return a+1
class T3(T): b = staticmethod(_g)
t = T3()
test_eq(t.b(2), 3)

Runtime type checking is handy, so let's make it easy!

@contextmanager
def working_directory(path):
    "Change working directory to `path` and return to previous on exit."
    prev_cwd = Path.cwd()
    os.chdir(path)
    try: yield
    finally: os.chdir(prev_cwd)
#def is_listy(x): return isinstance(x,(list,tuple,Generator))

add_docs[source]

add_docs(cls_doc=None, **docs)

Copy values from docs to cls docstrings, and confirm all public methods are documented

docs[source]

docs()

Decorator version of add_docs, using _docs dict

class _T:
    def f(self): pass
    @classmethod
    def g(cls): pass
add_docs(_T, "a", f="f", g="g")

test_eq(_T.__doc__, "a")
test_eq(_T.f.__doc__, "f")
test_eq(_T.g.__doc__, "g")

custom_dir[source]

custom_dir(c, add:list)

Implement custom __dir__, adding add to cls

is_iter[source]

is_iter(o)

Test whether o can be used in a for loop

assert is_iter([1])
assert not is_iter(array(1))
assert is_iter(array([1,2]))
assert (o for o in range(3))

class bind[source]

bind(fn, *pargs, **pkwargs)

Same as partial, except you can use arg0 arg1 etc param placeholders

def myfn(a,b,c,d=1,e=2): return(a,b,c,d,e)
test_eq(bind(myfn, arg1, 17, arg0, e=3)(19,14), (14,17,19,1,3))
test_eq(bind(myfn, 17, arg0, e=3)(19,14), (17,19,14,1,3))
test_eq(bind(myfn, 17, e=3)(19,14), (17,19,14,1,3))
test_eq(bind(myfn)(17,19,14), (17,19,14,1,2))
test_eq(bind(myfn, 17,19,14,e=arg0)(3), (17,19,14,1,3))

class GetAttr[source]

GetAttr()

Inherit from this to have all attr accesses in self._xtra passed down to self.default

Inherit from GetAttr to have attr access passed down to an instance attribute. This makes it easy to create composites that don't require callers to know about their components.

You can customise the behaviour of GetAttr in subclasses via;

  • _default
    • By default, this is set to 'default', so attr access is passed down to self.default
    • _default can be set to the name of any instance attribute that does not start with dunder __
  • _xtra
    • By default, this is None, so all attr access is passed down
    • You can limit which attrs get passed down by setting _xtra to a list of attribute names
class _C(GetAttr):
    # allow all attributes to get passed to `self.default` (by leaving _xtra=None)
    def __init__(self,a): self.default = a
    def foo(self): noop

t = _C('Hi')
test_eq(t.lower(), 'hi')
test_eq(t.upper(), 'HI')
assert 'lower' in dir(t)
assert 'upper' in dir(t)
class _C(GetAttr):
    _xtra = ['lower'] # specify which attributes get passed to `self.default`
    def __init__(self,a): self.default = a
    def foo(self): noop

t = _C('Hi')
test_eq(t.default, 'Hi')
test_eq(t.lower(), 'hi')
test_fail(lambda: t.upper())
assert 'lower' in dir(t)
assert 'upper' not in dir(t)
class _C(GetAttr):
    _default = '_data' # use different component name; `self._data` rather than `self.default`
    def __init__(self,a): self._data = a
    def foo(self): noop

t = _C('Hi')
test_eq(t._data, 'Hi')
test_eq(t.lower(), 'hi')
test_eq(t.upper(), 'HI')
assert 'lower' in dir(t)
assert 'upper' in dir(t)
class _C(GetAttr):
    _default = 'data' # use a bad component name; i.e. self.data does not exist
    def __init__(self,a): self.default = a
    def foo(self): noop
# TODO: should we raise an error when we create a new instance ...
t = _C('Hi')
test_eq(t.default, 'Hi')
# ... or is it enough for all GetAttr features to raise errors
test_fail(lambda: t.data)
test_fail(lambda: t.lower())
test_fail(lambda: t.upper())
test_fail(lambda: dir(t))
class B:
    def __init__(self): self.a = A()
@funcs_kwargs
class A(GetAttr):
    wif=after_iter= noops
    _methods = 'wif after_iter'.split()
    _default = 'dataset'
    def __init__(self, **kwargs): pass
a = A()
b = A(wif=a.wif)
#Failing test. TODO Jeremy, not sure what you were testing here
#a = A()
#b = A(wif=a.wif)
#tst = pickle.dumps(b)
#c = pickle.loads(tst)

delegate_attr[source]

delegate_attr(k, to)

Use in __getattr__ to delegate to attr to without inheriting from GetAttr

class _C:
    f = 'Hi'
    def __getattr__(self, k): return delegate_attr(self, k, 'f')

t = _C()
test_eq(t.lower(), 'hi')

coll_repr[source]

coll_repr(c, max_n=10)

String repr of up to max_n items of (possibly lazy) collection c

test_eq(coll_repr(range(1000), 5), '(#1000) [0,1,2,3,4...]')

mask2idxs[source]

mask2idxs(mask)

Convert bool mask or index list to index L

# just for tests
import torch
test_eq(mask2idxs([False,True,False,True]), [1,3])
test_eq(mask2idxs(array([False,True,False,True])), [1,3])
test_eq(mask2idxs(torch.tensor([False,True,False,True])), [1,3])
test_eq(mask2idxs(array([1,2,3])), [1,2,3])

class CollBase[source]

CollBase(items)

Base class for composing a list of items

cycle[source]

cycle(o)

Like itertools.cycle except creates list of Nones if o is empty

test_eq(itertools.islice(cycle([1,2,3]),5), [1,2,3,1,2])
test_eq(itertools.islice(cycle([]),3), [None]*3)
test_eq(itertools.islice(cycle(None),3), [None]*3)
test_eq(itertools.islice(cycle(1),3), [1,1,1])

zip_cycle[source]

zip_cycle(x, *args)

Like itertools.zip_longest but cycles through elements of all but first argument

test_eq(zip_cycle([1,2,3,4],list('abc')), [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'a')])

is_indexer[source]

is_indexer(idx)

Test whether idx will index a single item in a list

negate_func[source]

negate_func(f)

Create new function that negates result of f

def f(a): return a>0
test_eq(f(1),True)
test_eq(negate_func(f)(1),False)
test_eq(negate_func(f)(a=-1),True)

class L[source]

L(items=None, *rest, use_list=False, match=None) :: CollBase

Behaves like a list of items but can also index with list of indices or masks

You can create an L from an existing iterable (e.g. a list, range, etc) and access or modify it with an int list/tuple index, mask, int, or slice. All list methods can also be used with L.

t = L(range(12))
test_eq(t, list(range(12)))
test_ne(t, list(range(11)))
t.reverse()
test_eq(t[0], 11)
t[3] = "h"
test_eq(t[3], "h")
t[3,5] = ("j","k")
test_eq(t[3,5], ["j","k"])
test_eq(t, L(t))
test_eq(L(L(1,2),[3,4]), ([1,2],[3,4]))
t
(#12) [11,10,9,'j',7,'k',5,4,3,2...]

There are optimized indexers for arrays, tensors, and DataFrames.

arr = np.arange(9).reshape(3,3)
t = L(arr, use_list=None)
test_eq(t[1,2], arr[[1,2]])

arr = np.arange(9).reshape(3,3)
t = L(arr, use_list=None)
test_eq(t[1,2], arr[[1,2]])

df = pd.DataFrame({'a':[1,2,3]})
t = L(df, use_list=None)
test_eq(t[1,2], L(pd.DataFrame({'a':[2,3]}, index=[1,2]), use_list=None))

You can also modify an L with append, +, and *.

t = L()
test_eq(t, [])
t.append(1)
test_eq(t, [1])
t += [3,2]
test_eq(t, [1,3,2])
t = t + [4]
test_eq(t, [1,3,2,4])
t = 5 + t
test_eq(t, [5,1,3,2,4])
test_eq(L(1,2,3), [1,2,3])
test_eq(L(1,2,3), L(1,2,3))
t = L(1)*5
t = t.map(operator.neg)
test_eq(t,[-1]*5)
test_eq(~L([True,False,False]), L([False,True,True]))
t = L(range(4))
test_eq(zip(t, L(1).cycle()), zip(range(4),(1,1,1,1)))
t = L.range(100)
test_shuffled(t,t.shuffle())
def _f(x,a=0): return x+a
t = L(1)*5
test_eq(t.map(_f), t)
test_eq(t.map(_f,1), [2]*5)
test_eq(t.map(_f,a=2), [3]*5)

An L can be constructed from anything iterable, although tensors and arrays will not be iterated over on construction, unless you pass use_list to the constructor.

test_eq(L([1,2,3]),[1,2,3])
test_eq(L(L([1,2,3])),[1,2,3])
test_ne(L([1,2,3]),[1,2,])
test_eq(L('abc'),['abc'])
test_eq(L(range(0,3)),[0,1,2])
test_eq(L(o for o in range(0,3)),[0,1,2])
test_eq(L(array(0)),[array(0)])
test_eq(L([array(0),array(1)]),[array(0),array(1)])
test_eq(L(array([0.,1.1]))[0],array([0.,1.1]))
test_eq(L(array([0.,1.1]), use_list=True), [array(0.),array(1.1)])  # `use_list=True` to unwrap arrays/arrays

If match is not None then the created list is same len as match, either by:

  • If len(items)==1 then items is replicated,
  • Otherwise an error is raised if match and items are not already the same size.
test_eq(L(1,match=[1,2,3]),[1,1,1])
test_eq(L([1,2],match=[2,3]),[1,2])
test_fail(lambda: L([1,2],match=[1,2,3]))

If you create an L from an existing L then you'll get back the original object (since L uses the NewChkMeta metaclass).

test_is(L(t), t)

An L is considred equal to a list if they have the same elements. It's never considered equal to a str a set or a dict even if they have the same elements/keys.

test_eq(L(['a', 'b']), ['a', 'b'])
test_ne(L(['a', 'b']), 'ab')
test_ne(L(['a', 'b']), {'a', 'b'})
test_ne(L(['a', 'b']), {'a':1, 'b':2})

Methods

L.__getitem__[source]

L.__getitem__(idx)

Retrieve idx (can be list of indices, or mask, or int) items

t = L(range(12))
test_eq(t[1,2], [1,2])                # implicit tuple
test_eq(t[[1,2]], [1,2])              # list
test_eq(t[:3], [0,1,2])               # slice
test_eq(t[[False]*11 + [True]], [11]) # mask
test_eq(t[array(3)], 3)

L.__setitem__[source]

L.__setitem__(idx, o)

Set idx (can be list of indices, or mask, or int) items to o (which is broadcast if not iterable)

t[4,6] = 0
test_eq(t[4,6], [0,0])
t[4,6] = [1,2]
test_eq(t[4,6], [1,2])

L.unique[source]

L.unique()

Unique items, in stable order

test_eq(L(1,2,3,4,4).unique(), [1,2,3,4])

L.val2idx[source]

L.val2idx()

Dict from value to index

test_eq(L(1,2,3).val2idx(), {3:2,1:0,2:1})

L.filter[source]

L.filter(f, negate=False, **kwargs)

Create new L filtered by predicate f, passing args and kwargs to f

list(t)
[0, 1, 2, 3, 1, 5, 2, 7, 8, 9, 10, 11]
test_eq(t.filter(lambda o:o<5), [0,1,2,3,1,2])
test_eq(t.filter(lambda o:o<5, negate=True), [5,7,8,9,10,11])

L.argwhere[source]

L.argwhere(f, negate=False, **kwargs)

Like filter, but return indices for matching items

test_eq(t.argwhere(lambda o:o<5), [0,1,2,3,4,6])

L.map[source]

L.map(f, *args, **kwargs)

Create new L with f applied to all items, passing args and kwargs to f

test_eq(L.range(4).map(operator.neg), [0,-1,-2,-3])

If f is a string then it is treated as a format string to create the mapping:

test_eq(L.range(4).map('#{}#'), ['#0#','#1#','#2#','#3#'])

If f is a dictionary (or anything supporting __getitem__) then it is indexed to create the mapping:

test_eq(L.range(4).map(list('abcd')), list('abcd'))

If the special argument _arg is passed, then that is the kwarg used in the map.

#What is this? TODO Jeremy: fix
#L.range(4).map(f, b=arg0)
def f(a=None,b=None): return b
test_eq(L.range(4).map(f, b=arg0), range(4))

L.map_dict[source]

L.map_dict(f='noop', *args, **kwargs)

Like map, but creates a dict from items to function results

test_eq(L(range(1,5)).map_dict(), {1:1, 2:2, 3:3, 4:4})
test_eq(L(range(1,5)).map_dict(operator.neg), {1:-1, 2:-2, 3:-3, 4:-4})

L.zip[source]

L.zip(cycled=False)

Create new L with zip(*items)

t = L([[1,2,3],'abc'])
test_eq(t.zip(), [(1, 'a'),(2, 'b'),(3, 'c')])
t = L([[1,2,3,4],['a','b','c']])
test_eq(t.zip(cycled=True ), [(1, 'a'),(2, 'b'),(3, 'c'),(4, 'a')])
test_eq(t.zip(cycled=False), [(1, 'a'),(2, 'b'),(3, 'c')])

L.map_zip[source]

L.map_zip(f, *args, cycled=False, **kwargs)

Combine zip and starmap

t = L([1,2,3],[2,3,4])
test_eq(t.map_zip(operator.mul), [2,6,12])

L.zipwith[source]

L.zipwith(*rest, cycled=False)

Create new L with self zip with each of *rest

b = [[0],[1],[2,2]]
t = L([1,2,3]).zipwith(b)
test_eq(t, [(1,[0]), (2,[1]), (3,[2,2])])

L.map_zipwith[source]

L.map_zipwith(f, *rest, cycled=False, **kwargs)

Combine zipwith and starmap

test_eq(L(1,2,3).map_zipwith(operator.mul, [2,3,4]), [2,6,12])

L.itemgot[source]

L.itemgot(*idxs)

Create new L with item idx of all items

test_eq(t.itemgot(1), b)

L.attrgot[source]

L.attrgot(k, default=None)

Create new L with attr k of all items

a = [SimpleNamespace(a=3,b=4),SimpleNamespace(a=1,b=2)]
test_eq(L(a).attrgot('b'), [4,2])

L.sorted[source]

L.sorted(key=None, reverse=False)

New L sorted by key. If key is str then use attrgetter. If key is int then use itemgetter

test_eq(L(a).sorted('a').attrgot('b'), [2,4])

L.split[source]

L.split(s, sep=None, maxsplit=-1)

Same as str.split, but returns an L

test_eq(L.split('a b c'), list('abc'))

L.range[source]

L.range(a, b=None, step=None)

Same as range, but returns an L. Can pass a collection for a, to use len(a)

test_eq_type(L.range([1,1,1]), L(range(3)))
test_eq_type(L.range(5,2,2), L(range(5,2,2)))

L.concat[source]

L.concat()

Concatenate all elements of list

test_eq(L([0,1,2,3],4,L(5,6)).concat(), range(7))

L.copy[source]

L.copy()

Same as list.copy, but returns an L

t = L([0,1,2,3],4,L(5,6)).copy()
test_eq(t.concat(), range(7))