In [1]:
from tabulate import tabulate
import pandas as pd
from pathlib import Path
import json
import os
import altair as alt
from collections import defaultdict
from IPython.core.display import display, HTML
# alt.renderers.enable('html')
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')
In [2]:
VERSION = '0.4.1' # before tf.function compatibility
VERSION = '0.5.0' # after tf.function compatibility which cripple the perf currently
def load_benchmark_results(VERSION):
tables = defaultdict(lambda: defaultdict(dict))
res_dir = 'results/%s' % VERSION
results = []
for fname in Path(res_dir).glob('*.json'):
for l in open(fname):
d = json.loads(l)
res = {}
# flattenning
for k, v in d.items():
if k == 'timings':
for tn, tv in v.items():
if tn == 'raw':
continue
name = '%s_time' % (tn)
res[name] = tv
else:
res[k] = v
res['shape_str'] = "x".join([str(x) for x in d['shape']])
res['test_name'] = "%s%s" % (d['group'], d['name'])
tables[res['test_name']][res['shape_str']][res['backend']] = res['avg_time']
results.append(res)
return pd.DataFrame(results), tables
df, tables = load_benchmark_results(VERSION)
df.head()
Out[2]:
In [ ]:
In [3]:
def htmlout(data):
display(HTML(data))
def display_table(group_name):
BACKENDS = ['numpy', 'tensorflow-cpu', 'tensorflow-gpu', 'cupy']
headers = ['shape', 'numpy', 'tf-cpu', 'tf-gpu', 'cupy' ]
# htmlout('<h3>%s</h3>' % group_name)
for dim in ['1D', '2D', '3D']:
prob_name = group_name + dim
if prob_name in tables:
raw = tables[prob_name]
rows = []
for shape, data in tables[prob_name].items():
row = [shape]
baseline = data['numpy']
for b in BACKENDS:
ts = round(data[b], 3)
perf = round(baseline / (data[b] + 0.000001), 2)
val = "%s (%.2fx)" % (ts, perf)
row.append(val)
rows.append(row)
htmlout('<br/><b>%s</b>' % dim)
htmlout(tabulate(rows, headers=headers, tablefmt='html'))
In [4]:
def display_chart(group_name):
data = df[df['group']==group_name].sort_values('input_size')
chart = alt.Chart(data, title='').mark_bar().encode(
x=alt.X('backend', axis=alt.Axis(title='')),
y=alt.Y('avg_time', sort='-x', axis=alt.Axis(title='')),
color=alt.Color('backend', legend=None),
column=alt.Column('shape_str', title='', sort=alt.EncodingSortField(field='input_size', order='ascending'))
).resolve_scale(y='independent')
chart = chart.facet(column='name', spacing=50, title=group_name).configure_title(fontSize=20, offset=5, orient='top', anchor='middle')
return chart
In [5]:
display_chart('OneMax')
Out[5]:
In [6]:
display_table('OneMax')
In [7]:
display_chart('TSP')
Out[7]:
In [8]:
display_table('TSP')
In [9]:
display_chart('RandomMutation')
Out[9]:
In [10]:
display_table('RandomMutation')
In [11]:
display_chart('UniformCrossover')
Out[11]:
In [12]:
display_table('UniformCrossover')
In [13]:
display_chart('SingleCrossover')
Out[13]:
In [14]:
display_table('SingleCrossover')
In [15]:
display_chart('DualCrossover')
Out[15]:
In [16]:
display_table('DualCrossover')
In [17]:
display_chart('Reverse')
Out[17]:
In [18]:
display_table('Reverse')
In [19]:
display_chart('Shuffle')
Out[19]:
In [20]:
display_table('Shuffle')
In [21]:
display_chart('Fittest')
Out[21]:
In [22]:
display_table('Shuffle')
In [23]:
sorted(set(df[df['test_type']=='Selection']['group']))
Out[23]: