In [1]:
%load_ext autoreload
%autoreload 2
In [2]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *
import re
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import rcParams
%config InlineBackend.figure_format = 'retina'
import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")
In [5]:
exps = ['sigopt_test1', 'sigopt_test2', 'sigopt_test3']
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)
In [6]:
test_string = '0_learning_'
re.match('^\d+', test_string)[0]
Out[6]:
In [7]:
df.head(5)
Out[7]:
In [8]:
df.columns
Out[8]:
In [9]:
df['experiment_file_name'].unique()
Out[9]:
In [10]:
def fix_name(s):
if s == '/Users/lsouza/nta/results/sigopt_test1/experiment_state-2020-03-20_02-59-23.json':
return 'SigOpt-A'
elif s == '/Users/lsouza/nta/results/sigopt_test2/experiment_state-2020-03-20_18-07-05.json':
return "SigOpt-B"
elif s == '/Users/lsouza/nta/results/sigopt_test3/experiment_state-2020-03-25_01-36-27.json':
return 'Random Search'
df['experiment_file_name'] = df['experiment_file_name'].apply(fix_name)
In [11]:
df['experiment_file_name'].unique()
Out[11]:
In [12]:
def get_index(s):
return int(re.match('^\d+', s)[0])
df['index_pos'] = df['Experiment Name'].apply(get_index)
In [13]:
df['density'] = df['on_perc']
In [14]:
df.iloc[17]
Out[14]:
In [15]:
df.groupby('experiment_file_name')['model'].count()
Out[15]:
In [16]:
# helper functions
def mean_and_std(s):
return "{:.3f} ± {:.3f}".format(s.mean(), s.std())
def round_mean(s):
return "{:.0f}".format(round(s.mean()))
stats = ['min', 'max', 'mean', 'std']
def agg(columns, filter=None, round=3):
if filter is None:
return (df.groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
else:
return (df[filter].groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
In [17]:
agg(['experiment_file_name'])
Out[17]:
In [25]:
def plot_acc_over_time(plot_title):
plt.figure(figsize=(12,6))
df_plot = df[df['experiment_file_name'] == plot_title]
sns.lineplot(df_plot['index_pos'], y=df_plot['val_acc_last'])
plt.xticks(np.arange(0,300,30))
plt.ylim(0,0.80)
plt.title(plot_title)
In [26]:
# how to plot?
plot_acc_over_time('Random Search')
plot_acc_over_time('SigOpt-A')
plot_acc_over_time('SigOpt-B')
In [30]:
def accumulate(series):
series = list(series)
cum_series = [series[0]]
for i in range(1, len(series)):
cum_series.append(max(cum_series[i-1], series[i]))
return cum_series
def plot_best_acc_over_time(plot_title):
plt.figure(figsize=(12,6))
df_plot = df[df['experiment_file_name'] == plot_title].sort_values('index_pos')
df_plot['cum_acc'] = accumulate(df_plot['val_acc_last'])
sns.lineplot(df_plot['index_pos'], y=df_plot['cum_acc'])
plt.xticks(np.arange(0,301,30))
plt.ylim(0,0.80)
plt.title(plot_title)
In [32]:
plot_best_acc_over_time('Random Search')
plot_best_acc_over_time('SigOpt-A')
plot_best_acc_over_time('SigOpt-B')
In [37]:
# list top 5 values of each
# show best values
def show_best(experiment):
df_exp = df[df['experiment_file_name'] == experiment].sort_values('val_acc_last', ascending=False)[:5]
return df_exp[['index_pos', 'learning_rate', 'density', 'momentum', 'weight_decay', 'val_acc_last']]
In [38]:
show_best('Random Search')
Out[38]:
In [39]:
show_best('SigOpt-A')
Out[39]:
In [40]:
show_best('SigOpt-B')
Out[40]:
In [ ]: