In [1]:
%load_ext autoreload
%autoreload 2
In [21]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *
import re
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import rcParams
%config InlineBackend.figure_format = 'retina'
import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")
In [3]:
exps = ['sigopt_baseline_comp', 'test_sigopt.py']
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)
In [32]:
test_string = '0_learning_'
re.match('^\d+', test_string)[0]
Out[32]:
In [31]:
df.head(5)
Out[31]:
In [5]:
df.columns
Out[5]:
In [13]:
df['experiment_file_name'].unique()
Out[13]:
In [11]:
def fix_name(s):
if s == '/Users/lsouza/nta/results/sigopt_baseline_comp/experiment_state-2020-03-16_03-33-36.json':
return 'Random Search'
elif s == '/Users/lsouza/nta/results/test_sigopt.py/experiment_state-2020-03-15_23-03-55.json':
return "SigOpt-A"
elif s == '/Users/lsouza/nta/results/test_sigopt.py/experiment_state-2020-03-16_01-05-45.json':
return "SigOpt-B"
df['experiment_file_name'] = df['experiment_file_name'].apply(fix_name)
In [12]:
df['experiment_file_name'].unique()
Out[12]:
In [100]:
def get_index(s):
return int(re.match('^\d+', s)[0])
df['index_pos'] = df['Experiment Name'].apply(get_index)
In [143]:
df['density'] = df['on_perc']
In [101]:
df.iloc[17]
Out[101]:
In [39]:
df.groupby('experiment_file_name')['model'].count()
Out[39]:
In [40]:
# helper functions
def mean_and_std(s):
return "{:.3f} ± {:.3f}".format(s.mean(), s.std())
def round_mean(s):
return "{:.0f}".format(round(s.mean()))
stats = ['min', 'max', 'mean', 'std']
def agg(columns, filter=None, round=3):
if filter is None:
return (df.groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
else:
return (df[filter].groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
In [41]:
agg(['experiment_file_name'])
Out[41]:
In [109]:
def plot_acc_over_time(plot_title):
plt.figure(figsize=(12,6))
df_plot = df[df['experiment_file_name'] == plot_title]
sns.lineplot(df_plot['index_pos'], y=df_plot['val_acc_last'])
plt.xticks(np.arange(0,100,5))
plt.ylim(0.67,0.76)
plt.title(plot_title)
In [121]:
# how to plot?
plot_acc_over_time('Random Search')
plot_acc_over_time('SigOpt-A')
plot_acc_over_time('SigOpt-B')
In [141]:
def accumulate(series):
series = list(series)
cum_series = [series[0]]
for i in range(1, len(series)):
cum_series.append(max(cum_series[i-1], series[i]))
return cum_series
def plot_best_acc_over_time(plot_title):
plt.figure(figsize=(12,6))
df_plot = df[df['experiment_file_name'] == plot_title].sort_values('index_pos')
df_plot['cum_acc'] = accumulate(df_plot['val_acc_last'])
sns.lineplot(df_plot['index_pos'], y=df_plot['cum_acc'])
plt.xticks(np.arange(0,101,5))
plt.ylim(0.71,0.76)
plt.title(plot_title)
In [142]:
plot_best_acc_over_time('Random Search')
plot_best_acc_over_time('SigOpt-A')
plot_best_acc_over_time('SigOpt-B')
In [152]:
# list top 5 values of each
# show best values
def show_best(experiment):
df_exp = df[df['experiment_file_name'] == experiment].sort_values('val_acc_last', ascending=False)[:5]
return df_exp[['index_pos', 'learning_rate', 'density', 'val_acc_last']]
In [153]:
show_best('Random Search')
Out[153]:
In [154]:
show_best('SigOpt-A')
Out[154]:
In [155]:
show_best('SigOpt-B')
Out[155]:
In [ ]: