Notes from David Beazley's Python3 Metaprogramming tutorial (2013)

"ported" to Python 2.7, unless noted otherwise

A Debugging Decorator



In [7]:

    
from functools import wraps



In [8]:

    
def debug(func):
    msg = func.__name__
    # wraps is used to keep the metadata of the original function
    @wraps(func)
    def wrapper(*args, **kwargs):
        print(msg)
        return func(*args, **kwargs)
    return wrapper



In [9]:

    
@debug
def add(x,y):
    return x+y



In [10]:

    
add(2,3)









    



add






    Out[10]:





5



In [11]:

    
def add(x,y):
    return x+y



In [12]:

    
debug(add)









    Out[12]:





<function __main__.add>



In [13]:

    
debug(add)(2,3)









    



add






    Out[13]:





5

Decorators with arguments

Calling convention

@decorator(args)
def func():
    pass

Evaluation

func = decorator(args)(func)



In [14]:

    
def debug_with_args(prefix=''):
    def decorate(func):
        msg = prefix + func.__name__
        @wraps(func)
        def wrapper(*args, **kwargs):
            print(msg)
            return func(*args, **kwargs)
        return wrapper
    return decorate



In [15]:

    
@debug_with_args(prefix='***')
def mul(x,y):
    return x*y



In [16]:

    
mul(2,3)









    



***mul






    Out[16]:





6



In [17]:

    
def mul(x,y):
    return x*y



In [18]:

    
debug_with_args(prefix='***')









    Out[18]:





<function __main__.decorate>



In [19]:

    
debug_with_args(prefix='***')(mul)









    Out[19]:





<function __main__.mul>



In [20]:

    
debug_with_args(prefix='***')(mul)(2,3)









    



***mul






    Out[20]:





6

Decorators with arguments: a reformulation

TODO: show what happens without the partial application to itself!



In [21]:

    
from functools import wraps, partial

def debug_with_args2(func=None, prefix=''):
    if func is None: # no function was passed
        return partial(debug_with_args2, prefix=prefix)
    
    msg = prefix + func.__name__
    @wraps(func)
    def wrapper(*args, **kwargs):
        print(msg)
        return func(*args, **kwargs)
    return wrapper



In [22]:

    
@debug_with_args2(prefix='***')
def div(x,y):
    return x / y



In [23]:

    
div(4,2)









    



***div






    Out[23]:





2



In [24]:

    
def div(x,y):
    return x / y



In [25]:

    
debug_with_args2(prefix='***')









    Out[25]:





<functools.partial at 0x7f74b4169788>



In [26]:

    
debug_with_args2(prefix='***')(div)









    Out[26]:





<function __main__.div>



In [27]:

    
debug_with_args2(prefix='***')(div)(4,2)









    



***div






    Out[27]:





2



In [28]:

    
f = debug_with_args2(prefix='***')



In [29]:

    
def div(x,y):
    return x / y



In [30]:

    
debug_with_args2(prefix='***')(div)









    Out[30]:





<function __main__.div>

Debug with arguments: without partial()

this won't work with arguments



In [31]:

    
def debug_with_args_nonpartial(func, prefix=''):    
    msg = prefix + func.__name__
    @wraps(func)
    def wrapper(*args, **kwargs):
        print(msg)
        return func(*args, **kwargs)
    return wrapper



In [32]:

    
def plus1(x):
    return x+1



In [37]:

    
debug_with_args_nonpartial(plus1, prefix='***')(23)









    



***plus1






    Out[37]:





24



In [38]:

    
@debug_with_args_nonpartial
def plus1(x):
    return x+1



In [39]:

    
plus1(23)









    



plus1






    Out[39]:





24



In [40]:

    
@debug_with_args_nonpartial(prefix='***')
def plus1(x):
    return x+1









    



---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-40-7f57f207acb8> in <module>()
----> 1 @debug_with_args_nonpartial(prefix='***')
      2 def plus1(x):
      3     return x+1

TypeError: debug_with_args_nonpartial() takes at least 1 argument (1 given)

Decorators with arguments: memprof-style

this doesn't work at all

def memprof(*args, **kwargs):
    def inner(func):
        return MemProf(func, *args, **kwargs)

    # To allow @memprof with parameters
    if len(args) and callable(args[0]):
        func = args[0]
        args = args[1:]
        return inner(func)
    else:
        return inner



In [108]:

    
def debug_with_args3(*args, **kwargs):
    def inner(func, **kwargs):
        if 'prefix' in kwargs:
            msg = kwargs['prefix'] + func.__name__
        else:
            msg = func.__name__
        print(msg)
        return func

    # decorator without arguments
    if len(args) == 1 and callable(args[0]):
        func = args[0]
        return inner(func)
    # decorator with keyword arguments
    else:
        return partial(inner, prefix=kwargs['prefix'])



In [109]:

    
def plus2(x):
    return x+2



In [110]:

    
debug_with_args3(plus2)(23)









    



plus2






    Out[110]:





25



In [111]:

    
debug_with_args3(prefix='***')(plus2)(23)









    



***plus2






    Out[111]:





25



In [119]:

    
@debug_with_args3 # WRONG: this shouldn't print anything during creation
def plus2(x):
    return x+2









    



plus2



In [118]:

    
plus2(12) # WRONG: this should print the function name and the prefix









    Out[118]:





14



In [117]:

    
@debug_with_args3(prefix='###') # WRONG: this shouldn't print anything during creation
def plus2(x):
    return x+2









    



###plus2



In [116]:

    
plus2(12) # WRONG: this should print the function name and the prefix









    Out[116]:





14

Class decorators

decorate all methods of a class at once
NOTE: only instance methods will be wrapped, i.e. this won't work with static- or class methods



In [76]:

    
def debugmethods(cls):
    for name, val in vars(cls).items():
        if callable(val):
            setattr(cls, name, debug(val))
    return cls



In [85]:

    
@debugmethods
class Spam(object):
    def foo(self):
        pass
    def bar(self):
        pass



In [86]:

    
s = Spam()



In [87]:

    
s.foo()

foo



In [88]:

    
s.bar()

bar

Class decoration: debug access to attributes



In [96]:

    
def debugattr(cls):
    orig_getattribute = cls.__getattribute__
    
    def __getattribute__(self, name):
        print('Get:', name)
        return orig_getattribute(self, name)
    cls.__getattribute__ = __getattribute__
    
    return cls



In [97]:

    
@debugattr
class Ham(object):
    def foo(self):
        pass
    def bar(self):
        pass



In [105]:

    
h = Ham()



In [107]:

    
h.foo()









    



('Get:', 'foo')



In [108]:

    
h.bar









    



('Get:', 'bar')






    Out[108]:





<bound method Ham.bar of <__main__.Ham object at 0x7f4854a86110>>

Debug all the classes?

TODO: this looks Python3-specific

Solution: A Metaclass



In [112]:

    
class debugmeta(type):
    def __new__(cls, clsname, bases, clsdict):
        clsobj = super(cls).__new__(cls, clsname, bases, clsdict)
        clsobj = debugmethods(clsobj)
        return clsobj



In [117]:

    
# class Base(metaclass=debugmeta): # won't work in Python 2.7
#     pass

# class Bam(Base):
#     pass

# cf. minute 27

Can we inject the debugging code into all known classes?



In [126]:

    
class Spam:
    pass



In [127]:

    
s = Spam()



In [192]:

    
from copy import deepcopy

current_vars = deepcopy(globals())

for var in current_vars:
    if callable(current_vars[var]):
        print var,









    



---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-192-73c7d5d10a13> in <module>()
      1 from copy import deepcopy
      2 
----> 3 current_vars = deepcopy(globals())
      4 
      5 for var in current_vars:

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    188                             raise Error(
    189                                 "un(deep)copyable object of type %s" % cls)
--> 190                 y = _reconstruct(x, rv, 1, memo)
    191 
    192     memo[d] = y

/usr/lib/python2.7/copy.pyc in _reconstruct(x, info, deep, memo)
    332     if state:
    333         if deep:
--> 334             state = deepcopy(state, memo)
    335         if hasattr(y, '__setstate__'):
    336             y.__setstate__(state)

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_method(x, memo)
    262 
    263 def _deepcopy_method(x, memo): # Copy instance methods
--> 264     return type(x)(x.im_func, deepcopy(x.im_self, memo), x.im_class)
    265 _deepcopy_dispatch[types.MethodType] = _deepcopy_method
    266 

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    188                             raise Error(
    189                                 "un(deep)copyable object of type %s" % cls)
--> 190                 y = _reconstruct(x, rv, 1, memo)
    191 
    192     memo[d] = y

/usr/lib/python2.7/copy.pyc in _reconstruct(x, info, deep, memo)
    332     if state:
    333         if deep:
--> 334             state = deepcopy(state, memo)
    335         if hasattr(y, '__setstate__'):
    336             y.__setstate__(state)

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_list(x, memo)
    228     memo[id(x)] = y
    229     for a in x:
--> 230         y.append(deepcopy(a, memo))
    231     return y
    232 d[list] = _deepcopy_list

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    188                             raise Error(
    189                                 "un(deep)copyable object of type %s" % cls)
--> 190                 y = _reconstruct(x, rv, 1, memo)
    191 
    192     memo[d] = y

/usr/lib/python2.7/copy.pyc in _reconstruct(x, info, deep, memo)
    332     if state:
    333         if deep:
--> 334             state = deepcopy(state, memo)
    335         if hasattr(y, '__setstate__'):
    336             y.__setstate__(state)

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_inst(x, memo)
    296     else:
    297         state = x.__dict__
--> 298     state = deepcopy(state, memo)
    299     if hasattr(y, '__setstate__'):
    300         y.__setstate__(state)

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    188                             raise Error(
    189                                 "un(deep)copyable object of type %s" % cls)
--> 190                 y = _reconstruct(x, rv, 1, memo)
    191 
    192     memo[d] = y

/usr/lib/python2.7/copy.pyc in _reconstruct(x, info, deep, memo)
    332     if state:
    333         if deep:
--> 334             state = deepcopy(state, memo)
    335         if hasattr(y, '__setstate__'):
    336             y.__setstate__(state)

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    161     copier = _deepcopy_dispatch.get(cls)
    162     if copier:
--> 163         y = copier(x, memo)
    164     else:
    165         try:

/usr/lib/python2.7/copy.pyc in _deepcopy_dict(x, memo)
    255     memo[id(x)] = y
    256     for key, value in x.iteritems():
--> 257         y[deepcopy(key, memo)] = deepcopy(value, memo)
    258     return y
    259 d[dict] = _deepcopy_dict

/usr/lib/python2.7/copy.pyc in deepcopy(x, memo, _nil)
    188                             raise Error(
    189                                 "un(deep)copyable object of type %s" % cls)
--> 190                 y = _reconstruct(x, rv, 1, memo)
    191 
    192     memo[d] = y

/usr/lib/python2.7/copy.pyc in _reconstruct(x, info, deep, memo)
    327     if deep:
    328         args = deepcopy(args, memo)
--> 329     y = callable(*args)
    330     memo[id(x)] = y
    331 

/home/arne/.virtualenvs/notebook/lib/python2.7/copy_reg.pyc in __newobj__(cls, *args)
     91 
     92 def __newobj__(cls, *args):
---> 93     return cls.__new__(cls, *args)
     94 
     95 def _slotnames(cls):

TypeError: object.__new__(thread.lock) is not safe, use thread.lock.__new__()



In [190]:

    
frozendict









    



---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-190-d7a2856aac45> in <module>()
----> 1 from collections import frozendict

ImportError: cannot import name frozendict



In [189]:

    
for var in current_vars:
    cls = getattr(current_vars[var], '__class__')
    if cls:
        print var, cls









    



__ <type 'type'>






    



---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-189-cccfd502667e> in <module>()
----> 1 for var in current_vars:
      2     cls = getattr(current_vars[var], '__class__')
      3     if cls:
      4         print var, cls

RuntimeError: dictionary changed size during iteration



In [169]:

    
print current_vars['Spam']
type(current_vars['Spam'])









    



__main__.Spam






    Out[169]:





classobj



In [128]:

    
callable(Spam)









    Out[128]:





True



In [129]:

    
callable(s)









    Out[129]:





False



In [163]:

    
isinstance(Spam, classobj)









    



---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-163-71e284e54497> in <module>()
----> 1 isinstance(Spam, classobj)

NameError: name 'classobj' is not defined



In [170]:

    
__name__









    Out[170]:





'__main__'



In [179]:

    
sc = s.__class__



In [188]:

    
type('Foo', (), {})









    Out[188]:





__main__.Foo



In [ ]: