In [2]:

    

%pylab inline

import radical.synapse   as rs
import matplotlib.pyplot as plt
import pandas            as pd


    

    




Populating the interactive namespace from numpy and matplotlib

In [3]:

    

import radical.synapse.utils as rsu


    

In [8]:

    

cmd   = "mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro"
dburl = "file:///home/merzky/experiments/synapse-wrap-2015/data/"
f     = rs.utils.get_all_frames(cmd, url=dburl)
print f
f     = f.drop(['cmd', 'cmd_idx', 'mem_peak', 'tag_prob', 'tags', 'time_start', 'cpu_sequence', 'mem_sequence', 'sto_sequence'], axis=1)

print "number of data sets: %d" % len(f)


    

    




mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
skip command mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
   ! command mdrun -nt 1 -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
skip command mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
   ! command mdrun -nt 1 -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
skip command mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
   ! command mdrun -nt 1 -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
skip command mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
   ! command mdrun -nt 1 -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
skip command mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
   ! command mdrun -nt 1 -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
skip command mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
   ! command mdrun -nt 1 -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
mdrun -o traj.trr -e ener.edr -s topol.tpr -g mdlog.log -cpo state.cpt -c outgro
                                                    cmd  \
0     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
2     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
3     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
4     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
5     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
6     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
7     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
8     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
9     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
10    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
11    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
12    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
13    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
14    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
15    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
16    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
17    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
18    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
19    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
20    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
21    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
22    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
23    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
24    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
25    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
26    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
27    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
28    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
29    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
...                                                 ...   
1195  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1196  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1197  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1198  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1199  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1200  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1201  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1202  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1203  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1204  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1205  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1206  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1207  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1208  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1209  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1210  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1211  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1212  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1213  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1214  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1215  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1216  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1217  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1218  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1219  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1220  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1221  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1222  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1223  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   
1224  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...   

                                                cmd_idx     cpu_flops  \
0     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.128137e+09   
1     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.044926e+09   
2     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.114534e+09   
3     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.096882e+09   
4     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.066949e+09   
5     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.089691e+09   
6     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.063366e+09   
7     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.932840e+09   
8     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.155257e+09   
9     mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.151032e+09   
10    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.154190e+09   
11    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.124303e+09   
12    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.991711e+09   
13    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.946893e+09   
14    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.919050e+09   
15    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.148680e+09   
16    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.127160e+09   
17    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.196743e+09   
18    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.043999e+09   
19    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.493000e+09   
20    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.603924e+09   
21    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.720813e+09   
22    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.305242e+09   
23    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.339883e+09   
24    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.444482e+09   
25    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.524588e+09   
26    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.607661e+09   
27    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.351658e+09   
28    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.681300e+09   
29    mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.153574e+09   
...                                                 ...           ...   
1195  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.030528e+09   
1196  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.439581e+09   
1197  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.564477e+09   
1198  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.096636e+09   
1199  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.605291e+09   
1200  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.403216e+09   
1201  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.046644e+09   
1202  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.408958e+09   
1203  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.499771e+09   
1204  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.126308e+09   
1205  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  4.075051e+09   
1206  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.228895e+09   
1207  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.339406e+09   
1208  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.649850e+09   
1209  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.494437e+09   
1210  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.719206e+09   
1211  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.346990e+09   
1212  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.870821e+09   
1213  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.538110e+09   
1214  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.200693e+09   
1215  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.659282e+09   
1216  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.477665e+09   
1217  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.672757e+09   
1218  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.677070e+09   
1219  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.478181e+09   
1220  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.260630e+09   
1221  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.712336e+09   
1222  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.756056e+09   
1223  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  3.448551e+09   
1224  mdrun -o traj.trr -e ener.edr -s topol.tpr -g ...  2.698633e+09   

           cpu_ops                                       cpu_sequence  \
0     8.526550e+10  [[0.100187623, {u'real': 0.100187623, u'cycles...   
1     7.549052e+10  [[0.100167689, {u'real': 0.100167689, u'cycles...   
2     8.523115e+10  [[0.100228547, {u'real': 0.100228547, u'cycles...   
3     8.516328e+10  [[0.10020691, {u'real': 0.10020691, u'cycles':...   
4     7.552249e+10  [[0.100163272, {u'real': 0.100163272, u'cycles...   
5     7.550081e+10  [[0.100145211, {u'real': 0.100145211, u'cycles...   
6     7.561538e+10  [[0.100097759, {u'real': 0.100097759, u'cycles...   
7     7.560376e+10  [[0.10007696, {u'real': 0.10007696, u'cycles':...   
8     8.553395e+10  [[0.100242382, {u'real': 0.100242382, u'cycles...   
9     8.520053e+10  [[0.100238004, {u'real': 0.100238004, u'cycles...   
10    8.518613e+10  [[0.100219714, {u'real': 0.100219714, u'cycles...   
11    8.563763e+10  [[0.100277275, {u'real': 0.100277275, u'cycles...   
12    7.559303e+10  [[0.100145985, {u'real': 0.100145985, u'cycles...   
13    7.563125e+10  [[0.100141606, {u'real': 0.100141606, u'cycles...   
14    7.556878e+10  [[0.100147719, {u'real': 0.100147719, u'cycles...   
15    8.546819e+10  [[0.10026655, {u'real': 0.10026655, u'cycles':...   
16    8.525002e+10  [[0.100208255, {u'real': 0.100208255, u'cycles...   
17    8.523940e+10  [[0.100225718, {u'real': 0.100225718, u'cycles...   
18    7.555911e+10  [[0.100179928, {u'real': 0.100179928, u'cycles...   
19    7.405481e+08  [[5.000138709, {u'real': 5.000138709, u'cycles...   
20    1.593792e+09  [[5.000216063, {u'real': 5.000216063, u'cycles...   
21    1.621697e+09  [[5.000254713, {u'real': 5.000254713, u'cycles...   
22    7.154526e+08  [[5.000142433, {u'real': 5.000142433, u'cycles...   
23    7.321959e+08  [[5.000227781, {u'real': 5.000227781, u'cycles...   
24    7.469705e+08  [[5.000198211, {u'real': 5.000198211, u'cycles...   
25    7.528727e+08  [[5.00015518, {u'real': 5.00015518, u'cycles':...   
26    1.588271e+09  [[5.000210141, {u'real': 5.000210141, u'cycles...   
27    7.525373e+08  [[5.00019573, {u'real': 5.00019573, u'cycles':...   
28    1.607445e+09  [[5.000235635, {u'real': 5.000235635, u'cycles...   
29    7.052592e+08  [[5.000189655, {u'real': 5.000189655, u'cycles...   
...            ...                                                ...   
1195  7.560193e+10  [[10.000153818, {u'real': 10.000153818, u'cycl...   
1196  7.915084e+10  [[10.000283715, {u'real': 10.000283715, u'cycl...   
1197  7.935777e+10  [[10.000208133, {u'real': 10.000208133, u'cycl...   
1198  7.558324e+10  [[10.000173256, {u'real': 10.000173256, u'cycl...   
1199  7.936558e+10  [[10.000274135, {u'real': 10.000274135, u'cycl...   
1200  7.918487e+10  [[10.000226574, {u'real': 10.000226574, u'cycl...   
1201  7.559836e+10  [[10.000170714, {u'real': 10.000170714, u'cycl...   
1202  7.929496e+10  [[10.000235561, {u'real': 10.000235561, u'cycl...   
1203  7.940589e+10  [[10.000404205, {u'real': 10.000404205, u'cycl...   
1204  7.555490e+10  [[10.000172945, {u'real': 10.000172945, u'cycl...   
1205  7.555844e+10  [[10.000165804, {u'real': 10.000165804, u'cycl...   
1206  7.043315e+08  [[10.000191774, {u'real': 10.000191774, u'cycl...   
1207  7.330962e+08  [[10.000162878, {u'real': 10.000162878, u'cycl...   
1208  1.617193e+09  [[10.000184301, {u'real': 10.000184301, u'cycl...   
1209  7.322381e+08  [[10.000228204, {u'real': 10.000228204, u'cycl...   
1210  1.604903e+09  [[10.000271327, {u'real': 10.000271327, u'cycl...   
1211  7.376035e+08  [[10.00018978, {u'real': 10.00018978, u'cycles...   
1212  6.814783e+08  [[10.000276724, {u'real': 10.000276724, u'cycl...   
1213  1.567042e+09  [[10.000197093, {u'real': 10.000197093, u'cycl...   
1214  7.155885e+08  [[10.000158987, {u'real': 10.000158987, u'cycl...   
1215  1.605547e+09  [[10.000234997, {u'real': 10.000234997, u'cycl...   
1216  1.457035e+09  [[10.000239417, {u'real': 10.000239417, u'cycl...   
1217  1.586100e+09  [[10.000184353, {u'real': 10.000184353, u'cycl...   
1218  1.611800e+09  [[10.000204584, {u'real': 10.000204584, u'cycl...   
1219  7.678153e+08  [[10.000186032, {u'real': 10.000186032, u'cycl...   
1220  7.239186e+08  [[10.00016195, {u'real': 10.00016195, u'cycles...   
1221  1.613287e+09  [[10.000206747, {u'real': 10.000206747, u'cycl...   
1222  1.652393e+09  [[10.000203923, {u'real': 10.000203923, u'cycl...   
1223  7.521119e+08  [[10.00018749, {u'real': 10.00018749, u'cycles...   
1224  1.578544e+09  [[10.000236949, {u'real': 10.000236949, u'cycl...   

         host     mem_peak     mem_rss  \
0     thinkie  822472704.0  27209728.0   
1     thinkie   57643008.0   5795840.0   
2     thinkie   75141120.0         NaN   
3     thinkie   75821056.0         NaN   
4     thinkie   57643008.0   5799936.0   
5     thinkie   57643008.0   5795840.0   
6     thinkie   57643008.0   5795840.0   
7     thinkie   57643008.0   5791744.0   
8     thinkie  754855936.0  26718208.0   
9     thinkie  755036160.0  26894336.0   
10    thinkie  756494336.0  28233728.0   
11    thinkie  820858880.0  25604096.0   
12    thinkie   57643008.0   5795840.0   
13    thinkie   57643008.0   5799936.0   
14    thinkie   59183104.0   6909952.0   
15    thinkie  821821440.0  26570752.0   
16    thinkie  819531776.0  24129536.0   
17    thinkie  754712576.0  26566656.0   
18    thinkie   59183104.0   6909952.0   
19    thinkie   56799232.0   2301952.0   
20    thinkie   64815104.0  11952128.0   
21    thinkie   64811008.0  11890688.0   
22    thinkie   56840192.0   2748416.0   
23    thinkie   56840192.0   3010560.0   
24    thinkie   56840192.0   2748416.0   
25    thinkie   56799232.0   2031616.0   
26    thinkie   64815104.0  11956224.0   
27    thinkie   56799232.0   2486272.0   
28    thinkie   64811008.0  11886592.0   
29    thinkie   56799232.0   2035712.0   
...       ...          ...         ...   
1195  thinkie   57643008.0   5804032.0   
1196  thinkie  744792064.0  16359424.0   
1197  thinkie  811900928.0  16363520.0   
1198  thinkie   57643008.0   5799936.0   
1199  thinkie  744792064.0  16367616.0   
1200  thinkie  744792064.0  16367616.0   
1201  thinkie   57643008.0   5799936.0   
1202  thinkie  811900928.0  16367616.0   
1203  thinkie  744787968.0  16363520.0   
1204  thinkie   57643008.0   5804032.0   
1205  thinkie   57643008.0   5804032.0   
1206  thinkie   56840192.0   3010560.0   
1207  thinkie   56799232.0   2297856.0   
1208  thinkie   64815104.0  11972608.0   
1209  thinkie   56799232.0   2297856.0   
1210  thinkie   64815104.0  11952128.0   
1211  thinkie   56799232.0   1228800.0   
1212  thinkie   56799232.0   2306048.0   
1213  thinkie   64815104.0  11956224.0   
1214  thinkie   56799232.0   2301952.0   
1215  thinkie   64815104.0  11960320.0   
1216  thinkie  207835136.0  12107776.0   
1217  thinkie   64815104.0  11960320.0   
1218  thinkie   64815104.0  11960320.0   
1219  thinkie   56799232.0   2031616.0   
1220  thinkie   56799232.0   2301952.0   
1221  thinkie   64815104.0  11960320.0   
1222  thinkie   64815104.0  11960320.0   
1223  thinkie   56799232.0   2297856.0   
1224  thinkie   64811008.0  11890688.0   

                                           mem_sequence     mem_size mode  \
0     [[0.00750088691711, {u'rss': 22831104.0, u'siz...  755363840.0  emu   
1     [[0.176950931549, {u'rss': 2301952.0, u'size':...   57643008.0  pro   
2                                                    []          NaN  emu   
3                                                    []          NaN  emu   
4     [[0.175516843796, {u'rss': 2035712.0, u'size':...   57643008.0  pro   
5     [[0.171715974808, {u'rss': 2297856.0, u'size':...   57643008.0  pro   
6     [[0.177594900131, {u'rss': 3014656.0, u'size':...   57643008.0  pro   
7     [[0.174451112747, {u'rss': 1757184.0, u'size':...   57643008.0  pro   
8     [[0.00567483901978, {u'rss': 22413312.0, u'siz...  754855936.0  emu   
9     [[0.00756192207336, {u'rss': 22601728.0, u'siz...  755036160.0  emu   
10    [[0.00763320922852, {u'rss': 23838720.0, u'siz...  756494336.0  emu   
11    [[0.0080041885376, {u'rss': 21315584.0, u'size...  753750016.0  emu   
12    [[0.174223899841, {u'rss': 2301952.0, u'size':...   57643008.0  pro   
13    [[0.171927928925, {u'rss': 2031616.0, u'size':...   57643008.0  pro   
14    [[0.172309875488, {u'rss': 2297856.0, u'size':...   59183104.0  pro   
15    [[0.00746393203735, {u'rss': 22278144.0, u'siz...  754712576.0  emu   
16    [[0.0081889629364, {u'rss': 19886080.0, u'size...  752422912.0  emu   
17    [[0.00675106048584, {u'rss': 22286336.0, u'siz...  754712576.0  emu   
18    [[0.176671981812, {u'rss': 3010560.0, u'size':...   59183104.0  pro   
19    [[0.172960042953, {u'rss': 2301952.0, u'size':...   56799232.0  pro   
20    [[0.00780892372131, {u'rss': 11952128.0, u'siz...   64815104.0  emu   
21    [[0.00769901275635, {u'rss': 11890688.0, u'siz...   64811008.0  emu   
22    [[0.17431306839, {u'rss': 2748416.0, u'size': ...   56840192.0  pro   
23    [[0.180746078491, {u'rss': 3010560.0, u'size':...   56840192.0  pro   
24    [[0.176115989685, {u'rss': 2748416.0, u'size':...   56840192.0  pro   
25    [[0.173825979233, {u'rss': 2031616.0, u'size':...   56799232.0  pro   
26    [[0.0123560428619, {u'rss': 11956224.0, u'size...   64811008.0  emu   
27    [[0.175159931183, {u'rss': 2486272.0, u'size':...   56705024.0  pro   
28    [[0.00717496871948, {u'rss': 11886592.0, u'siz...   64811008.0  emu   
29    [[0.174281835556, {u'rss': 2035712.0, u'size':...   56799232.0  pro   
...                                                 ...          ...  ...   
1195  [[0.178366184235, {u'rss': 2306048.0, u'size':...   57643008.0  pro   
1196  [[0.009850025177, {u'rss': 12046336.0, u'size'...  744792064.0  emu   
1197  [[0.00867509841919, {u'rss': 12079104.0, u'siz...  744792064.0  emu   
1198  [[0.175328969955, {u'rss': 2031616.0, u'size':...   57643008.0  pro   
1199  [[0.0105979442596, {u'rss': 12083200.0, u'size...  744792064.0  emu   
1200  [[0.0122940540314, {u'rss': 11972608.0, u'size...  744792064.0  emu   
1201  [[0.178199052811, {u'rss': 2031616.0, u'size':...   57643008.0  pro   
1202  [[0.00917100906372, {u'rss': 11833344.0, u'siz...  744792064.0  emu   
1203  [[0.00893592834473, {u'rss': 11927552.0, u'siz...  744787968.0  emu   
1204  [[0.175380945206, {u'rss': 2306048.0, u'size':...   57643008.0  pro   
1205  [[0.175154209137, {u'rss': 2031616.0, u'size':...   57643008.0  pro   
1206  [[0.175178050995, {u'rss': 3010560.0, u'size':...   56840192.0  pro   
1207  [[0.175376176834, {u'rss': 2297856.0, u'size':...   56799232.0  pro   
1208  [[0.011824131012, {u'rss': 11972608.0, u'size'...   64815104.0  emu   
1209  [[0.172039985657, {u'rss': 2297856.0, u'size':...   56799232.0  pro   
1210  [[0.00945997238159, {u'rss': 11952128.0, u'siz...   64815104.0  emu   
1211  [[0.171648025513, {u'rss': 1228800.0, u'size':...   56799232.0  pro   
1212  [[0.176815986633, {u'rss': 2306048.0, u'size':...   56799232.0  pro   
1213  [[0.00842809677124, {u'rss': 11956224.0, u'siz...   64815104.0  emu   
1214  [[0.175328016281, {u'rss': 2301952.0, u'size':...   56799232.0  pro   
1215  [[0.00855994224548, {u'rss': 11960320.0, u'siz...   64815104.0  emu   
1216  [[0.00898098945618, {u'rss': 12107776.0, u'siz...  140726272.0  emu   
1217  [[0.00943303108215, {u'rss': 11960320.0, u'siz...   64811008.0  emu   
1218  [[0.0117008686066, {u'rss': 11960320.0, u'size...   64811008.0  emu   
1219  [[0.177948951721, {u'rss': 2031616.0, u'size':...   56799232.0  pro   
1220  [[0.17592382431, {u'rss': 2301952.0, u'size': ...   56799232.0  pro   
1221  [[0.00992512702942, {u'rss': 11960320.0, u'siz...   64815104.0  emu   
1222  [[0.0127341747284, {u'rss': 11960320.0, u'size...   64815104.0  emu   
1223  [[0.174749135971, {u'rss': 2297856.0, u'size':...   56799232.0  pro   
1224  [[0.00641202926636, {u'rss': 11890688.0, u'siz...   64811008.0  emu   

      sto_read                                       sto_sequence  sto_write  \
0     103850.0  [[0.00579595565796, {u'read': 5577.0, u'write'...   510333.0   
1      25320.0  [[0.17525100708, {u'read': 11940.0, u'write': ...   397596.0   
2     104060.0  [[0.00394797325134, {u'read': 5577.0, u'write'...   510543.0   
3     104048.0  [[0.00435280799866, {u'read': 5577.0, u'write'...   510531.0   
4      25320.0  [[0.173436880112, {u'read': 11940.0, u'write':...   397597.0   
5      25320.0  [[0.170449018478, {u'read': 11940.0, u'write':...   397597.0   
6      25320.0  [[0.1719789505, {u'read': 11940.0, u'write': 0...   396829.0   
7      25320.0  [[0.175943136215, {u'read': 11940.0, u'write':...   396828.0   
8     104036.0  [[0.00434303283691, {u'read': 5577.0, u'write'...   508471.0   
9     104016.0  [[0.00595283508301, {u'read': 5577.0, u'write'...   510507.0   
10    104028.0  [[0.00591516494751, {u'read': 5577.0, u'write'...   510511.0   
11    103838.0  [[0.00380897521973, {u'read': 5577.0, u'write'...   510313.0   
12     25320.0  [[0.172652006149, {u'read': 11940.0, u'write':...   396443.0   
13     25320.0  [[0.169960021973, {u'read': 11940.0, u'write':...   397596.0   
14     25320.0  [[0.170670032501, {u'read': 11940.0, u'write':...   397596.0   
15    103862.0  [[0.00388598442078, {u'read': 16.0, u'write': ...   508297.0   
16    103854.0  [[0.00461196899414, {u'read': 5577.0, u'write'...   508289.0   
17    104052.0  [[0.00524091720581, {u'read': 5577.0, u'write'...   510535.0   
18     25320.0  [[0.171548843384, {u'read': 11940.0, u'write':...   397213.0   
19     11940.0  [[0.171128988266, {u'read': 11940.0, u'write':...        0.0   
20      5577.0  [[0.00502896308899, {u'read': 5577.0, u'write'...        0.0   
21      4745.0  [[0.00363111495972, {u'read': 4745.0, u'write'...        0.0   
22     11940.0  [[0.172340154648, {u'read': 11940.0, u'write':...        0.0   
23     11940.0  [[0.175515890121, {u'read': 11940.0, u'write':...        0.0   
24     11940.0  [[0.172415018082, {u'read': 11940.0, u'write':...        0.0   
25     11940.0  [[0.172706842422, {u'read': 11940.0, u'write':...        0.0   
26      5577.0  [[0.0053608417511, {u'read': 5577.0, u'write':...        0.0   
27     11940.0  [[0.171357870102, {u'read': 11940.0, u'write':...        0.0   
28      5577.0  [[0.00465083122253, {u'read': 5577.0, u'write'...        0.0   
29     11940.0  [[0.17339682579, {u'read': 11940.0, u'write': ...        0.0   
...        ...                                                ...        ...   
1195   25320.0  [[0.176710128784, {u'read': 11940.0, u'write':...   218652.0   
1196  282484.0  [[0.00706219673157, {u'read': 5577.0, u'write'...   218943.0   
1197  282492.0  [[0.00685095787048, {u'read': 5577.0, u'write'...   218951.0   
1198   25320.0  [[0.17317700386, {u'read': 11940.0, u'write': ...   222108.0   
1199  282492.0  [[0.00831294059753, {u'read': 5577.0, u'write'...   218951.0   
1200  282492.0  [[0.00358700752258, {u'read': 5577.0, u'write'...   218951.0   
1201   25320.0  [[0.177180051804, {u'read': 11940.0, u'write':...   219419.0   
1202  282492.0  [[0.00774097442627, {u'read': 5577.0, u'write'...   218951.0   
1203   64372.0  [[0.00439596176147, {u'read': 5577.0, u'write'...      831.0   
1204   25320.0  [[0.172008037567, {u'read': 11940.0, u'write':...   223644.0   
1205   25320.0  [[0.172609090805, {u'read': 11940.0, u'write':...   221340.0   
1206   11940.0  [[0.17283987999, {u'read': 11940.0, u'write': ...        0.0   
1207   11940.0  [[0.172183036804, {u'read': 11940.0, u'write':...        0.0   
1208    5577.0  [[0.00515007972717, {u'read': 5577.0, u'write'...        0.0   
1209   11940.0  [[0.169663906097, {u'read': 11940.0, u'write':...        0.0   
1210    5577.0  [[0.00372695922852, {u'read': 5577.0, u'write'...        0.0   
1211   11940.0  [[0.173236846924, {u'read': 11940.0, u'write':...        0.0   
1212   11940.0  [[0.174363136292, {u'read': 11940.0, u'write':...        0.0   
1213    5577.0  [[0.00366592407227, {u'read': 5577.0, u'write'...        0.0   
1214   11940.0  [[0.171993017197, {u'read': 11940.0, u'write':...        0.0   
1215    5577.0  [[0.00363397598267, {u'read': 5577.0, u'write'...        0.0   
1216    5577.0  [[0.00356888771057, {u'read': 5577.0, u'write'...        0.0   
1217    5577.0  [[0.00391817092896, {u'read': 5577.0, u'write'...        0.0   
1218    5577.0  [[0.00498700141907, {u'read': 5577.0, u'write'...        0.0   
1219   11940.0  [[0.176824092865, {u'read': 11940.0, u'write':...        0.0   
1220   11940.0  [[0.17399096489, {u'read': 11940.0, u'write': ...        0.0   
1221    5577.0  [[0.00370192527771, {u'read': 5577.0, u'write'...        0.0   
1222    5577.0  [[0.00502800941467, {u'read': 5577.0, u'write'...        0.0   
1223   11940.0  [[0.171761035919, {u'read': 11940.0, u'write':...        0.0   
1224    5577.0  [[0.00379705429077, {u'read': 5577.0, u'write'...        0.0   

      tag_prob  tag_rate   tag_step  \
0          1.0      10.0  1000000.0   
1          1.0      10.0  1000000.0   
2          1.0      10.0  1000000.0   
3          1.0      10.0  1000000.0   
4          1.0      10.0  1000000.0   
5          1.0      10.0  1000000.0   
6          1.0      10.0  1000000.0   
7          1.0      10.0  1000000.0   
8          1.0      10.0  1000000.0   
9          1.0      10.0  1000000.0   
10         1.0      10.0  1000000.0   
11         1.0      10.0  1000000.0   
12         1.0      10.0  1000000.0   
13         1.0      10.0  1000000.0   
14         1.0      10.0  1000000.0   
15         1.0      10.0  1000000.0   
16         1.0      10.0  1000000.0   
17         1.0      10.0  1000000.0   
18         1.0      10.0  1000000.0   
19         1.0       0.2    10000.0   
20         1.0       0.2    10000.0   
21         1.0       0.2    10000.0   
22         1.0       0.2    10000.0   
23         1.0       0.2    10000.0   
24         1.0       0.2    10000.0   
25         1.0       0.2    10000.0   
26         1.0       0.2    10000.0   
27         1.0       0.2    10000.0   
28         1.0       0.2    10000.0   
29         1.0       0.2    10000.0   
...        ...       ...        ...   
1195       1.0       0.1  1000000.0   
1196       1.0       0.1  1000000.0   
1197       1.0       0.1  1000000.0   
1198       1.0       0.1  1000000.0   
1199       1.0       0.1  1000000.0   
1200       1.0       0.1  1000000.0   
1201       1.0       0.1  1000000.0   
1202       1.0       0.1  1000000.0   
1203       1.0       0.1  1000000.0   
1204       1.0       0.1  1000000.0   
1205       1.0       0.1  1000000.0   
1206       1.0       0.1    10000.0   
1207       1.0       0.1    10000.0   
1208       1.0       0.1    10000.0   
1209       1.0       0.1    10000.0   
1210       1.0       0.1    10000.0   
1211       1.0       0.1    10000.0   
1212       1.0       0.1    10000.0   
1213       1.0       0.1    10000.0   
1214       1.0       0.1    10000.0   
1215       1.0       0.1    10000.0   
1216       1.0       0.1    10000.0   
1217       1.0       0.1    10000.0   
1218       1.0       0.1    10000.0   
1219       1.0       0.1    10000.0   
1220       1.0       0.1    10000.0   
1221       1.0       0.1    10000.0   
1222       1.0       0.1    10000.0   
1223       1.0       0.1    10000.0   
1224       1.0       0.1    10000.0   

                                                   tags  time_real  \
0     {u'step': u'1000000', u'rate': u'10', u'prob':...  40.065798   
1     {u'step': u'1000000', u'rate': u'10', u'prob':...  18.663017   
2     {u'step': u'1000000', u'rate': u'10', u'prob':...  40.307297   
3     {u'step': u'1000000', u'rate': u'10', u'prob':...  40.614254   
4     {u'step': u'1000000', u'rate': u'10', u'prob':...  18.569812   
5     {u'step': u'1000000', u'rate': u'10', u'prob':...  18.461251   
6     {u'step': u'1000000', u'rate': u'10', u'prob':...  18.609050   
7     {u'step': u'1000000', u'rate': u'10', u'prob':...  19.223703   
8     {u'step': u'1000000', u'rate': u'10', u'prob':...  39.686193   
9     {u'step': u'1000000', u'rate': u'10', u'prob':...  39.609140   
10    {u'step': u'1000000', u'rate': u'10', u'prob':...  39.544394   
11    {u'step': u'1000000', u'rate': u'10', u'prob':...  40.313282   
12    {u'step': u'1000000', u'rate': u'10', u'prob':...  18.937501   
13    {u'step': u'1000000', u'rate': u'10', u'prob':...  19.162227   
14    {u'step': u'1000000', u'rate': u'10', u'prob':...  19.282426   
15    {u'step': u'1000000', u'rate': u'10', u'prob':...  39.777070   
16    {u'step': u'1000000', u'rate': u'10', u'prob':...  40.076924   
17    {u'step': u'1000000', u'rate': u'10', u'prob':...  38.802631   
18    {u'step': u'1000000', u'rate': u'10', u'prob':...  18.684254   
19    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.212009   
20    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.612073   
21    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.596034   
22    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.216460   
23    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.219228   
24    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.216860   
25    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.213606   
26    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.609079   
27    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.224527   
28    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.599502   
29    {u'step': u'10000', u'rate': u'0.2', u'prob': ...   0.223638   
...                                                 ...        ...   
1195  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  18.757324   
1196  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  17.828447   
1197  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  17.385950   
1198  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  18.450077   
1199  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  17.233566   
1200  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  17.983419   
1201  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  18.681743   
1202  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  17.984965   
1203  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  17.646651   
1204  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  18.310532   
1205  {u'step': u'1000000', u'rate': u'0.1', u'prob'...  18.541714   
1206  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.218134   
1207  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.219529   
1208  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.610296   
1209  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.209544   
1210  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.590210   
1211  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.220378   
1212  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.237381   
1213  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.617405   
1214  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.223573   
1215  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.603752   
1216  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.588068   
1217  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.593432   
1218  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.602076   
1219  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.220752   
1220  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.222018   
1221  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.594796   
1222  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.599550   
1223  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.218095   
1224  {u'step': u'10000', u'rate': u'0.1', u'prob': ...   0.584942   

        time_start  
0     1.431894e+09  
1     1.431759e+09  
2     1.431894e+09  
3     1.431894e+09  
4     1.431759e+09  
5     1.431759e+09  
6     1.431759e+09  
7     1.431759e+09  
8     1.431894e+09  
9     1.431894e+09  
10    1.431894e+09  
11    1.431894e+09  
12    1.431759e+09  
13    1.431759e+09  
14    1.431759e+09  
15    1.431894e+09  
16    1.431894e+09  
17    1.431894e+09  
18    1.431759e+09  
19    1.431738e+09  
20    1.431889e+09  
21    1.431889e+09  
22    1.431738e+09  
23    1.431738e+09  
24    1.431738e+09  
25    1.431738e+09  
26    1.431889e+09  
27    1.431738e+09  
28    1.431889e+09  
29    1.431738e+09  
...            ...  
1195  1.431741e+09  
1196  1.431893e+09  
1197  1.431892e+09  
1198  1.431741e+09  
1199  1.431892e+09  
1200  1.431892e+09  
1201  1.431741e+09  
1202  1.431892e+09  
1203  1.431892e+09  
1204  1.431741e+09  
1205  1.431741e+09  
1206  1.431738e+09  
1207  1.431738e+09  
1208  1.431889e+09  
1209  1.431738e+09  
1210  1.431889e+09  
1211  1.431738e+09  
1212  1.431738e+09  
1213  1.431889e+09  
1214  1.431738e+09  
1215  1.431889e+09  
1216  1.431889e+09  
1217  1.431889e+09  
1218  1.431889e+09  
1219  1.431738e+09  
1220  1.431738e+09  
1221  1.431889e+09  
1222  1.431889e+09  
1223  1.431738e+09  
1224  1.431889e+09  

[1225 rows x 20 columns]
number of data sets: 1225

In [23]:

    

host_frames = dict()

steps = sorted(set(f['tag_step']))
rates = sorted(set(f['tag_rate']))
hosts = sorted(set(f['host']), reverse=True)
modes = ['pro', 'emu', 'exe']

# step are int
steps = [int(x) for x in steps]
f['tag_step'] = f['tag_step'].astype(int) 
print "steps: %s" % steps
print "rates: %s" % rates
print "hosts: %s" % hosts
print "modes: %s" % modes

for host in hosts:
    host_frames[host] = dict()
    host_frames[host]['all'] = f[f['host'] == host]
    for mode in modes:
        host_frames[host][mode] = f[(f['tag_rate'] == 1) & (f['mode'] == mode) & (f['host'] == host)]


    

    




steps: [10000, 50000, 100000, 500000, 1000000, 5000000, 10000000]
rates: [0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0]
hosts: ['thinkie', 'stampede', 'archer']
modes: ['pro', 'emu', 'exe']

In [ ]:

    




    

In [24]:

    

# define a colorscheme we use consistently throughout this notebook
colors   = {'exe'  : 'green'    , 
            'emu'  : 'blue'     , 
            'pro'  : 'red'      ,
            'diff' : 'black'    }
modnames = {'exe'  : 'execution', 
            'emu'  : 'emulation', 
            'pro'  : 'profiling'}

font = {'family' : 'monospace',
        'weight' : 'normal',
        'size'   : 14}

plt.rc('font', **font)

def create_figure(title=None):
    """
    create a plot object and tune its layout to our liking.
    """
    
    fig, ax = plt.subplots(figsize=(10,5))
    if title:
        ax.set_title(title, fontsize=18)
    
    ax.grid(False)

    ax.spines['right'].set_color('none')
    ax.spines['top'].set_color('none')

    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_ticks_position('left')

    ax.spines['bottom'].set_position(('axes', -0.05))
    ax.spines['left'].set_position(('axes', -0.02))
    ax.xaxis.grid(False)
    ax.yaxis.grid(False)

    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    
    fig.tight_layout()

    return ax


    

In [25]:

    

# \subsubsection{P.2 -- Profiling Consistency}

# We performed repeated profiling of the same application instances, in
# the same environment.  While the non-zero standard deviation
# indicates some noise in the measured metrics, the distribution is in
# very good agreement with the distribution of the pure application TTC
# (see figure~\ref{...}), which indicates the influence of system
# background.  The figure shows the same expected profiling consistency
# over a range of application sizes and sampling rates.

ax_ops = create_figure('CPU Operations over Sample Frequency and Problem Sizes')
labels = list()
host   = 'thinkie'

for mode in ['pro']:
  
    for step in steps:
        
        tmp = f[(f['tag_step'] == step) & (f['mode'] == mode) & (f['host'] == host)]
        tmp = tmp[['cpu_ops', 'tag_rate']].groupby('tag_rate')
        
        tmp_means  = tmp.mean()
        tmp_errors = tmp.std()
        
        # we do not have data for some rates, due to limited db size
        if not len(tmp_means):
            continue

        tmp_means.plot(logy=True, logx=True, yerr=tmp_errors, ax=ax_ops, capsize=10, marker='o')
        labels.append ('%8s iterations' % step)

ax_ops.set_xlabel("Sampling Frequency (Hz)", fontsize=14)
ax_ops.set_ylabel("Number of Operations",    fontsize=14)
legend(labels, loc='upper right', fontsize=12)
savefig('pro_ops.png', bbox_inches='tight')
savefig('pro_ops.pdf', bbox_inches='tight')


    

In [26]:

    

# ---------------------------------------------------------------------------------
# CPU ops

ax_ops = create_figure('Total CPU Operations over Sample and Problem Sizes')
labels = list()
host   = 'thinkie'
mode   = 'pro'
  
for rate in rates:
    
    tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
    tmp = tmp[['cpu_ops', 'tag_step']].groupby('tag_step')
        
    tmp_means  = tmp.mean()
    tmp_errors = tmp.std()
        
    # we do not have data for some rates, due to limited db size
    if not len(tmp_means):
        continue

    tmp_means.plot(logy=True, logx=True, yerr=tmp_errors, ax=ax_ops, capsize=10, color=colors[mode])
    labels.append ('%s (%4s Hz)' % (modnames[mode], rate))

ax_ops.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax_ops.set_ylabel("Number of Operations", fontsize=14)
plt.legend(labels, loc='best', fontsize=12)
plt.savefig('pro_cpu.png', bbox_inches='tight')
plt.savefig('pro_cpu.pdf', bbox_inches='tight')


    

In [27]:

    

# ---------------------------------------------------------------------------------
# memory

ax_mem = create_figure('Profiled Memory Usage over Sample and Problem Sizes')
labels = list()

for rate in rates:
    
    tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
    tmp = tmp[['mem_rss', 'tag_step']].groupby('tag_step')
    
    tmp_means  = tmp.mean()
    tmp_errors = tmp.std()
        
    # we do not have data for some rates, due to limited db size
    if not len(tmp_means):
        continue

    tmp_means.plot(logy=True, logx=True, yerr=tmp_errors, ax=ax_mem, capsize=10)
    labels.append ('%s (%4s Hz)' % (modnames[mode], rate))


ax_mem.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax_mem.set_ylabel("Resident Memory (Bytes)", fontsize=14)
plt.legend(labels, loc='best', fontsize=12)
plt.savefig('pro_mem.png', bbox_inches='tight')
plt.savefig('pro_mem.pdf', bbox_inches='tight')


# ---------------------------------------------------------------------------------
# sto read

ax_read = create_figure('Profiled Disk Read over Sample and Problem Sizes')
labels = list()

for rate in rates:
    
    tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
    tmp = tmp[['sto_read', 'tag_step']].groupby('tag_step')
    
    tmp_means  = tmp.mean()
    tmp_errors = tmp.std()
        
    # we do not have data for some rates, due to limited db size
    if not len(tmp_means):
        continue

    tmp_means.plot(logy=True, logx=True, yerr=tmp_errors, ax=ax_read, capsize=10)
    labels.append ('%s (%4s Hz)' % (modnames[mode], rate))

ax_read.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax_read.set_ylabel("Read from Storage (Bytes)", fontsize=14)
plt.legend(labels, loc='best', fontsize=12)
plt.savefig('pro_read.png', bbox_inches='tight')
plt.savefig('pro_read.pdf', bbox_inches='tight')


# ---------------------------------------------------------------------------------
# sto write

ax_write = create_figure('Profiled Disk Write over Sample and Problem Sizes')
labels = list()

for rate in rates:
    
    tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
    tmp = tmp[['sto_write', 'tag_step']].groupby('tag_step')
    
    tmp_means  = tmp.mean()
    tmp_errors = tmp.std()
        
    # we do not have data for some rates, due to limited db size
    if not len(tmp_means):
        continue

    tmp_means.plot(logy=True, logx=True, yerr=tmp_errors, ax=ax_write, capsize=10)
    labels.append ('%s (%4s Hz)' % (modnames[mode], rate))

ax_write.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax_write.set_ylabel("Write to Storage (Bytes)",  fontsize=14)
plt.legend(labels, loc='best', fontsize=12)
plt.savefig('pro_write.png', bbox_inches='tight')
plt.savefig('pro_write.pdf', bbox_inches='tight')


    

In [83]:

    

#  \subsubsection{P.1 -- Profiling Overhead}
#
# Figure~\ref{...} compares the TTC for two cases: pure application runs,
# and the execution of the same application on the same resource under
# the synapse profiler.  That measurement is shown for different
# problem sizes (application runtimes) and different sampling rates.
# The graph shows that the overhead introduced by the profiling is 
# negligible and remains so for the investigated range of problem 
# sizes and sampling rates.

host    = 'thinkie'
labels  = list()
host    = 'thinkie'

ax = create_figure('Profiling vs. Execution')

for mode in ['pro', 'exe']:
  
    for rate in rates:
    
        tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
        tmp = tmp[['time_real', 'tag_step']].groupby('tag_step')
    
        tmp_means  = tmp.mean()
        tmp_errors = tmp.std()
        
        # we do not have data for some rates, due to limited db size
        if not len(tmp_means):
            continue

        tmp_means.plot(logy=True, logx=True, yerr=tmp_errors, ax=ax, color=colors[mode], capsize=10)
        
        if mode == 'exe':
            labels.append (mode)
        else:
            labels.append ('%s (%4s Hz)' % (modnames[mode], rate))

ax.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax.set_ylabel("$T_x$ (seconds)", fontsize=14)
plt.legend(labels, loc='best', fontsize=12)
plt.savefig('pro_overhead.png', bbox_inches='tight')
plt.savefig('pro_overhead.pdf', bbox_inches='tight')


    

In [84]:

    

#  \subsubsection{P.1 -- Profiling Overhead}
#
# Figure~\ref{...} compares the TTC for two cases: pure application runs,
# and the execution of the same application on the same resource under
# the synapse profiler.  That measurement is shown for different
# problem sizes (application runtimes) and different sampling rates.
# The graph shows that the overhead introduced by the profiling is 
# negligible and remains so for the investigated range of problem 
# sizes and sampling rates.

my_labels = list()
my_colors = list()
host      = 'thinkie'

ax = create_figure('Profiling vs. Execution')

df = pd.DataFrame()
#for mode in ['pro', 'exe']:
for mode in ['pro', 'exe']:
    
    tmp_rates = rates
    if mode == 'exe':
        tmp_rates = [1]

    for rate in tmp_rates:
        
        idx = '%s (%4s Hz)' % (modnames[mode], rate)
        if mode == 'exe':
            idx = 'execution'

        tmpf = f[(f['host'] == host) & (f['mode'] == mode)][['time_real', 'tag_step', 'tag_rate']]

      # print tmpf
        tmp = tmpf[(tmpf['tag_rate'] == rate)].groupby('tag_step')
        means = tmp.mean()
        df[idx] = means['time_real']

        my_labels.append (idx)
        my_colors.append (colors[mode])

df.plot(logx=False, logy=True, kind='bar', ax=ax, color=my_colors, edgecolor=['black'], linewidth=1.0
)

ax.set_xticklabels(['$10^4$', '$5x10^4$', '$10^5$', '$5x10^5$', '$10^6$', '$5x10^6$', '$10^7$'], rotation=0)
ax.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax.set_ylabel("$T_x$ (seconds)", fontsize=14)
plt.legend(my_labels, loc='best', fontsize=12)
plt.savefig('pro_overhead_bars.png', bbox_inches='tight')
plt.savefig('pro_overhead_bars.pdf', bbox_inches='tight')


    

In [85]:

    

# \subsubsection{P.3/E.1 -- Profiling as Emulation Input}
# 
# The ultimate purpose of synapse's profiling is to feed synapse's
# emulation.  Figure~\ref{...} compares the TTC of pure application
# execution versus emulated application runs, on the very machine used
# for application profiling.  The graphs show that emulation tends to 
# incur an overhead, specifically if very high sampling rates are used.
# That can be explained by the added communication overhead between
# the sampling threads and processes, which is additional to the 
# profiled resource consumption of the application.

host   = 'thinkie'
labels = list()

ax1 = create_figure('Emulation Overhead (log/log)')


for mode in ['exe', 'emu']:
  
    for rate in rates:
    
        tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
        tmp = tmp[['time_real', 'tag_step']].groupby('tag_step')
        
        tmp_means  = tmp.mean()
        tmp_errors = tmp.std()
        
        if not len(tmp_means):
            continue

        tmp_means.plot(logy=True,  logx=True,  yerr=tmp_errors, ax=ax1, capsize=10, color=colors[mode])
        labels.append ('%s (%4s Hz)' % (modnames[mode], rate))

ax1.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax1.set_ylabel("$T_x$ (seconds)",                  fontsize=14)
plt.legend(labels, loc='best', fontsize=12)

# same w/o log

labels=list()
ax2 = create_figure('Emulation Overhead')
for mode in ['exe', 'emu']:
  
    for rate in rates:
    
        tmp = f[(f['tag_rate'] == rate) & (f['mode'] == mode) & (f['host'] == host)]
        tmp = tmp[['time_real', 'tag_step']].groupby('tag_step')
        
        tmp_means  = tmp.mean()
        tmp_errors = tmp.std()
        
        if not len(tmp_means):
            continue

        tmp_means.plot(logy=False, logx=False, yerr=tmp_errors, ax=ax2, capsize=10)
        labels.append ('%s (%4s Hz)' % (modnames[mode], rate))

ax2.set_xlabel("Application Configuration (#iterations)", fontsize=14)
ax2.set_ylabel("$T_x$ (seconds)", fontsize=14)
plt.legend(labels, loc='best', fontsize=12)

plt.savefig('emu_overhead.png', bbox_inches='tight')
plt.savefig('emu_overhead.pdf', bbox_inches='tight')


    

In [86]:

    

# \subsubsection{P.4/E.1 -- Profiling as Emulation Input}
# 
# Similar to figure~\ref{...}, figure~\ref{...} compares the
# application execution and application emulation on resources
# \I{different} than the one used for application profiling, ie. on
# Stampede and Archer.  Again the TTCs for application execution
# and emulation are resambling the application characteristics 
# over a range of application sizes and sampling rates.
#
# The plots though also show that the emulation overhead is 
# significant for small application sizes, ie. for application 
# runtimes of about one second or smaller.  We find that acceptable, 
# as our research focus lies on much longer living applications, and
# we assume that sub-second application runs will remain the exception
# in both HTDC and HPDC\F{acronyms}.

for host in hosts:
    emuf = host_frames[host]['emu']
    emu  = emuf[['time_real', 'tag_step']].groupby('tag_step')
        
    emu_means  = emu.mean()
    emu_errors = emu.std()

    exef = host_frames[host]['exe']
    exe  = exef[['time_real', 'tag_step']].groupby('tag_step')

    exe_means  = exe.mean()
    exe_errors = exe.std()

    ax = create_figure('Emulation vs. Execution (%s)' % host)
    ax.set_xlabel("Application Configuration (#iterations)", fontsize=14)
    ax.set_ylabel("$T_x$ (seconds)", fontsize=14)
    
    labels = list()
    handles = list()

    print_delta=True
    if print_delta:
        delta = 100 * abs(exe_means['time_real'] - emu_means['time_real']) / exe_means['time_real']
        delta = delta.clip(upper=100)
    
        ax.plot(1, color=colors['diff']) # NOTE: this is to get the diff label right :(
        h = delta.plot(logx=True, logy=False, label='diff', ax=ax, color=colors['diff'], secondary_y=True)
        handles.append(h)
        
        ax.right_ax.set_ylabel("$T_x$ diff (%)", fontsize=14)
        ax.right_ax.set_ylim((0,150))

        labels.append('diff (%)')

    
    emu_means.plot(logx=True, logy=True,  label='emu',  ax=ax, color=colors['emu'], yerr=emu_errors, capsize=10)
    exe_means.plot(logx=True, logy=True,  label='exe',  ax=ax, color=colors['exe'], yerr=exe_errors, capsize=10)
    labels.extend(['execution', 'emulation'])
    
    ax.legend(labels, loc='upper left', fontsize=12)

    plt.savefig('emu_vs_exe_%s.png' % host, bbox_inches='tight')
    plt.savefig('emu_vs_exe_%s.pdf' % host, bbox_inches='tight')


    

In [11]: