In [1]:
%autosave 10
Look inside the optimizing subfolder of this folder.
Also there's a massive 80 page handout. Handout contains the bulk of material for this tutorial, so run through
He didn't get past profiling, so look at the handout and type it all up into this IPython Notebook. Make it comprehensive.
Usually improving big-O of algorithms and data structures is the best first step.
Hardware is cheaper than programmer time.
In [4]:
import test.pystone
test.pystone.main()
In [12]:
%load optimizing/measuring/profile_me.py
the %%prun magic incantation shows output in the bottom pane of your IPython Notebook.
In [13]:
%%prun
# file profile_me.py
"""Example to be profiled.
"""
import time
def fast():
"""Wait 0.001 seconds.
"""
time.sleep(1e-3)
def slow():
"""Wait 0.1 seconds.
"""
time.sleep(0.1)
def use_fast():
"""Call `fast` 100 times.
"""
for _ in xrange(100):
fast()
def use_slow():
"""Call `slow` 100 times.
"""
for _ in xrange(100):
slow()
if __name__ == '__main__':
use_fast()
use_slow()
In [23]:
%timeit slow()
In [24]:
%timeit fast()
In [20]:
import cProfile
In [21]:
cProfile.runctx("slow()", globals(), locals())
In [36]:
cProfile.run("slow()")
In [22]:
cProfile.runctx("fast()", globals(), locals())
In [37]:
cProfile.run("use_fast()", "optimizing/fast.stats")
import pstats
stats = pstats.Stats("optimizing/fast.stats")
stats.print_stats()
Out[37]:
Or sort by time
In [38]:
stats.sort_stats("time").print_stats()
Out[38]:
Or print out who is calling a certain function:
In [39]:
stats.print_callers("fast")
Out[39]:
Or who is calling you:
In [40]:
stats.print_callees("use_fast")
Out[40]:
In [33]:
profiler = cProfile.Profile()
In [34]:
profiler.runcall(slow)
In [35]:
profiler.print_stats()
timeit module's timer by default gives the CPU time, in particular timeit.default_timer().cProfiler can be used to do both, and by default measures wall-clock time.
In [41]:
%load optimizing/measuring/cpu_time.py
In [44]:
# file: cpu_time.py
"""Measuring CPU time instead of wall clock time.
"""
import cProfile
import os
import sys
import time
# Make it work with Python 2 and Python 3.
if sys.version_info[0] < 3:
range = xrange
# This is important for Python 2.
def cpu_time():
"""Function for cpu time. Os dependent.
"""
if sys.platform == 'win32':
return os.times()[0]
else:
return time.clock()
def sleep():
"""Wait 2 seconds.
"""
time.sleep(2)
def count():
"""100 million loops.
"""
for _ in range(int(1e8)):
1 + 1
def test():
"""Run functions
"""
sleep()
count()
def clock_check():
"""Profile with wall clock and cpu time.
"""
# wall clock time (first print block)
profiler = cProfile.Profile()
profiler.run('test()')
profiler.print_stats()
# cpu time (second print block)
profiler = cProfile.Profile(cpu_time)
profiler.run('test()')
profiler.print_stats()
if __name__ == '__main__':
clock_check()
In [52]:
!cat optimizing/measuring/profile_me_use_line_profiler.py
!kernprof.py -l -v optimizing/measuring/profile_me_use_line_profiler.py
In [51]:
!cat optimizing/measuring/accumulate.py
!kernprof.py -l -v optimizing/measuring/accumulate.py
In [50]:
!cat optimizing/measuring/calc.py
!kernprof.py -l -v optimizing/measuring/calc.py
If you're willing to accept a performance hit you can catch the NameError exception and define an empty profile decorator, and let your program be executable without kernprof.py.
In [53]:
try:
@profile
def dummy():
"""Needs to be here to avoid syntax error"""
pass
except NameError:
def profile(func):
"""Empty decorator if not under kernprof.py"""
return func
In [56]:
!cat optimizing/measuring/local_ref.py
!kernprof.py -l -v optimizing/measuring/local_ref.py
!kernprof.py -v optimizing/measuring/local_ref.py
In [62]:
from guppy import hpy
h = hpy()
In [68]:
h.heap()
Out[68]:
In [69]:
biglist = range(1000000)
In [70]:
h.heap()
Out[70]:
In [72]:
h.heap()[0]
Out[72]:
Here are some command-line examples of hpy. We've written our own decorator to track the memory change resulting from a function call.
In [74]:
!cat optimizing/measuring/memory_size_hpy.py
!python optimizing/measuring/memory_size_hpy.py
And again we can use the same kernprof.py empty profile decorator as before if we wanted to no-op this decorator in code.
In [76]:
!cat optimizing/measuring/memory_growth_hpy.py
!python optimizing/measuring/memory_growth_hpy.py
make_big is very small because CPython uses a reference-counted synchronous garbage collector. The unused value is immediately garbage collected.
In [81]:
!cat optimizing/measuring/memory_growth_pympler.py
!python optimizing/measuring/memory_growth_pympler.py
!!AI why is make_big empty?
In [94]:
import sys
from pympler.asizeof import asizeof, flatsize
def list_mem(length, size_func=flatsize):
"""Measure incremental memory increase of a growing list.
"""
my_list= []
mem = [size_func(my_list)]
for elem in xrange(length):
my_list.append(elem)
mem.append(size_func(my_list))
return mem
SIZE = 1000
SHOW = 20
In [95]:
plot(list_mem(SIZE, size_func=flatsize))
Out[95]:
In [96]:
plot(list_mem(SIZE, size_func=asizeof))
Out[96]:
In [97]:
plot(list_mem(SIZE, size_func=sys.getsizeof))
Out[97]:
sys.getsizeof is the same thing as pympler.flatsize.
In [102]:
%load optimizing/measuring/list_alloc_steps.py
In [ ]:
# file: list_alloc_steps.py
"""Measure the number of memory allocation steps for a list.
"""
import sys
from pympler.asizeof import flatsize
def list_steps(lenght, size_func=sys.getsizeof):
"""Measure the number of memory alloaction steps for a list.
"""
my_list = []
steps = 0
int_size = size_func(int())
old_size = size_func(my_list)
for elem in xrange(lenght):
my_list.append(elem)
new_size = sys.getsizeof(my_list)
if new_size - old_size > int_size:
steps += 1
old_size = new_size
return steps
if __name__ == '__main__':
print 'Using sys.getsizeof:'
for size in [10, 100, 1000, 10000, int(1e5), int(1e6), int(1e7)]:
print '%10d: %3d' % (size, list_steps(size))
print 'Using pympler.asizeof.flatsize:'
for size in [10, 100, 1000, 10000, int(1e5), int(1e6), int(1e7)]:
print '%10d: %3d' % (size, list_steps(size, flatsize))
In [103]:
!python optimizing/measuring/list_alloc_steps.py
There's a line memory profiler, look in the handout.
In [ ]: